package api_logs import ( "net/url" "os" "strconv" "strings" "time" "github.com/aws/aws-lambda-go/events" ) var ( MaxReqBodyLength int = 1024 MaxResBodyLength int = 1024 ) func init() { if s := os.Getenv("API_LOGS_MAX_REQ_BODY_LENGTH"); s != "" { if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 { MaxReqBodyLength = int(i64) } } if s := os.Getenv("API_LOGS_MAX_RES_BODY_LENGTH"); s != "" { if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 { MaxResBodyLength = int(i64) } } } //var producer queues.Producer //func Init(p queues.Producer) { // producer = p //} //Call this at the end of an API request handler to capture the req/res as well as all actions taken during the processing //(note: action list is only reset when this is called - so must be called after each handler, else action list has to be reset at the start) func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) error { //if producer == nil { // return errors.Errorf("api_logs queue producer not set") //} //todo: filter out some noisy (method+path) //logs.Debugf("claim: %+v", claim) endTime := time.Now() var authType string var authUsername string if req.RequestContext.Identity.CognitoAuthenticationType != "" { authType = "cognito" split := strings.Split(req.RequestContext.Identity.CognitoAuthenticationProvider, ":") if len(split) > 0 { authUsername = split[len(split)-1] //= part after last ':' } } else { authType = "iam" split := strings.Split(req.RequestContext.Identity.UserArn, ":user/") if len(split) > 0 { authUsername = split[len(split)-1] //= part after ':user/' } } userID, _ := claim["UserID"].(int64) username, _ := claim["Username"].(string) accountID, _ := claim["AccountID"].(int64) if accountID == 0 { if accountIDParam, ok := req.QueryStringParameters["account_id"]; ok { if i64, err := strconv.ParseInt(accountIDParam, 10, 64); err == nil && i64 > 0 { accountID = i64 } } } apiLog := ApiLog{ StartTime: startTime, EndTime: endTime, DurMs: endTime.Sub(startTime).Milliseconds(), Type: "api-incoming", Method: req.HTTPMethod, Address: req.RequestContext.DomainName, Path: req.Path, ResponseCode: res.StatusCode, RequestID: requestID, InitialAuthType: authType, InitialAuthUsername: authUsername, SourceIP: req.RequestContext.Identity.SourceIP, UserAgent: req.RequestContext.Identity.UserAgent, UserID: userID, Username: username, AccountID: accountID, Request: ApiLogRequest{ Headers: req.Headers, QueryParameters: req.QueryStringParameters, BodySize: len(req.Body), //see below: Body: req.Body, }, Response: ApiLogResponse{ Headers: res.Headers, BodySize: len(res.Body), //see below: Body: res.Body, }, } if apiLog.Request.BodySize > MaxReqBodyLength { apiLog.Request.Body = req.Body[:MaxReqBodyLength] + "..." } else { apiLog.Request.Body = req.Body } if apiLog.Response.BodySize > MaxResBodyLength { apiLog.Response.Body = res.Body[:MaxResBodyLength] + "..." } else { apiLog.Response.Body = res.Body } //also copy multi-value query parameters to the log as CSV array values for n, as := range req.MultiValueQueryStringParameters { apiLog.Request.QueryParameters[n] = "[" + strings.Join(as, ",") + "]" } //todo: filter out excessive req/res body content per (method+path) //todo: also need to do for all actions... // if apiLog.Method == http.MethodGet { // apiLog.Response.Body = "<not logged>" // } //todo: filter out sensitive values (e.g. OTP) //if _, err := producer.NewEvent("API_LOGS"). // Type("api-log"). // RequestID(apiLog.RequestID). // Send(apiLog); err != nil { // return errors.Wrapf(err, "failed to send api-log") //} return nil } //LogIncomingAPIRequest() //Call LogOutgoingAPIRequest() after calling an API end-point as part of a handler, //to capture the details //and add it to the current handler log story for reporting/metrics func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, responseBody string, responseCode int) error { //if producer == nil { // return errors.Errorf("api_logs queue producer not set") //} //todo: filter out some noisy (method+path) //logs.Debugf("claim: %+v", claim) endTime := time.Now() userID, _ := claim["UserID"].(int64) username, _ := claim["Username"].(string) accountID, _ := claim["AccountID"].(int64) params := map[string]string{} parsedURL, err := url.Parse(urlString) if err == nil { for n, v := range parsedURL.Query() { params[n] = strings.Join(v, ",") } } apiLog := ApiLog{ StartTime: startTime, EndTime: endTime, DurMs: endTime.Sub(startTime).Milliseconds(), Type: "api-outgoing", Method: method, Path: parsedURL.Path, Address: parsedURL.Host, ResponseCode: responseCode, RequestID: requestID, UserID: userID, Username: username, AccountID: accountID, Request: ApiLogRequest{ //Headers: req.Headers, QueryParameters: params, BodySize: len(requestBody), //See below: Body: requestBody, }, Response: ApiLogResponse{ //Headers: res.Headers, BodySize: len(responseBody), //See below: Body: responseBody, }, } if apiLog.Request.BodySize > MaxReqBodyLength { apiLog.Request.Body = requestBody[:MaxReqBodyLength] + "..." } else { apiLog.Request.Body = requestBody } if apiLog.Response.BodySize > MaxResBodyLength { apiLog.Response.Body = responseBody[:MaxResBodyLength] + "..." } else { apiLog.Response.Body = responseBody } //todo: filter out sensitive values (e.g. OTP) //if _, err := producer.NewEvent("API_LOGS"). // Type("api-log"). // RequestID(apiLog.RequestID). // Send(apiLog); err != nil { // return errors.Wrapf(err, "failed to send api-log") //} return nil } //LogOutgoingAPIRequest() //ApiLog is the SQS event details struct encoded as JSON document, sent to SQS, to be logged for each API handler executed. type ApiLog struct { StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` DurMs int64 `json:"duration_ms"` //duration in milliseconds Type string `json:"type"` //incoming-api or outgoing-api Method string `json:"method"` Address string `json:"address"` //server address for incoming and outgoing Path string `json:"path"` ResponseCode int `json:"response_code"` RequestID string `json:"request_id"` InitialAuthUsername string `json:"initial_auth_username,omitempty"` InitialAuthType string `json:"initial_auth_type,omitempty"` AccountID int64 `json:"account_id,omitempty"` UserID int64 `json:"user_id,omitempty"` Username string `json:"username,omitempty"` SourceIP string `json:"source_ip,omitempty"` //only logged for incoming API UserAgent string `json:"user_agent,omitempty"` //only for incoming, indicate type of browser when UI RelevantID string `json:"relevant_id,omitempty"` Request ApiLogRequest `json:"request"` Response ApiLogResponse `json:"response"` } type ApiLogRequest struct { Headers map[string]string `json:"headers,omitempty"` QueryParameters map[string]string `json:"query_parameters,omitempty"` BodySize int `json:"body_size" search:"long"` //set even when body is truncated/omitted Body string `json:"body,omitempty"` //json body as a string } type ApiLogResponse struct { Headers map[string]string `json:"headers,omitempty"` BodySize int `json:"body_size"` //set even when body is truncated/omitted Body string `json:"body,omitempty"` //json content as a string }