Skip to content
Snippets Groups Projects
Commit 892d7326 authored by Francé Wilke's avatar Francé Wilke
Browse files

Api logs - get by ID

parent d4e877ef
No related branches found
No related tags found
No related merge requests found
......@@ -2,7 +2,6 @@ package api_logs
import (
"net/url"
"os"
"strconv"
"strings"
"time"
......@@ -10,40 +9,7 @@ import (
"github.com/aws/aws-lambda-go/events"
)
var (
MaxReqBodyLength int = 1024
MaxResBodyLength int = 1024
)
func init() {
if s := os.Getenv("API_LOGS_MAX_REQ_BODY_LENGTH"); s != "" {
if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 {
MaxReqBodyLength = int(i64)
}
}
if s := os.Getenv("API_LOGS_MAX_RES_BODY_LENGTH"); s != "" {
if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 {
MaxResBodyLength = int(i64)
}
}
}
//var producer queues.Producer
//func Init(p queues.Producer) {
// producer = p
//}
//Call this at the end of an API request handler to capture the req/res as well as all actions taken during the processing
//(note: action list is only reset when this is called - so must be called after each handler, else action list has to be reset at the start)
func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) error {
//if producer == nil {
// return errors.Errorf("api_logs queue producer not set")
//}
//todo: filter out some noisy (method+path)
//logs.Debugf("claim: %+v", claim)
func GenerateIncomingAPILog(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) ApiLog {
endTime := time.Now()
var authType string
......@@ -93,57 +59,24 @@ func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[stri
Headers: req.Headers,
QueryParameters: req.QueryStringParameters,
BodySize: len(req.Body),
//see below: Body: req.Body,
Body: req.Body,
},
Response: ApiLogResponse{
Headers: res.Headers,
BodySize: len(res.Body),
//see below: Body: res.Body,
Body: res.Body,
},
}
if apiLog.Request.BodySize > MaxReqBodyLength {
apiLog.Request.Body = req.Body[:MaxReqBodyLength] + "..."
} else {
apiLog.Request.Body = req.Body
}
if apiLog.Response.BodySize > MaxResBodyLength {
apiLog.Response.Body = res.Body[:MaxResBodyLength] + "..."
} else {
apiLog.Response.Body = res.Body
}
//also copy multi-value query parameters to the log as CSV array values
for n, as := range req.MultiValueQueryStringParameters {
apiLog.Request.QueryParameters[n] = "[" + strings.Join(as, ",") + "]"
}
//todo: filter out excessive req/res body content per (method+path)
//todo: also need to do for all actions...
// if apiLog.Method == http.MethodGet {
// apiLog.Response.Body = "<not logged>"
// }
//todo: filter out sensitive values (e.g. OTP)
//if _, err := producer.NewEvent("API_LOGS").
// Type("api-log").
// RequestID(apiLog.RequestID).
// Send(apiLog); err != nil {
// return errors.Wrapf(err, "failed to send api-log")
//}
return nil
} //LogIncomingAPIRequest()
//Call LogOutgoingAPIRequest() after calling an API end-point as part of a handler,
//to capture the details
//and add it to the current handler log story for reporting/metrics
func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, responseBody string, responseCode int) error {
//if producer == nil {
// return errors.Errorf("api_logs queue producer not set")
//}
//todo: filter out some noisy (method+path)
//logs.Debugf("claim: %+v", claim)
return apiLog
}
func GenerateOutgoingAPILog(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, requestHeaders map[string]string, responseBody string, responseCode int) ApiLog {
endTime := time.Now()
userID, _ := claim["UserID"].(int64)
username, _ := claim["Username"].(string)
......@@ -170,38 +103,20 @@ func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[stri
Username: username,
AccountID: accountID,
Request: ApiLogRequest{
//Headers: req.Headers,
Headers: requestHeaders,
QueryParameters: params,
BodySize: len(requestBody),
//See below: Body: requestBody,
Body: requestBody,
},
Response: ApiLogResponse{
//Headers: res.Headers,
Headers: requestHeaders,
BodySize: len(responseBody),
//See below: Body: responseBody,
Body: responseBody,
},
}
if apiLog.Request.BodySize > MaxReqBodyLength {
apiLog.Request.Body = requestBody[:MaxReqBodyLength] + "..."
} else {
apiLog.Request.Body = requestBody
return apiLog
}
if apiLog.Response.BodySize > MaxResBodyLength {
apiLog.Response.Body = responseBody[:MaxResBodyLength] + "..."
} else {
apiLog.Response.Body = responseBody
}
//todo: filter out sensitive values (e.g. OTP)
//if _, err := producer.NewEvent("API_LOGS").
// Type("api-log").
// RequestID(apiLog.RequestID).
// Send(apiLog); err != nil {
// return errors.Wrapf(err, "failed to send api-log")
//}
return nil
} //LogOutgoingAPIRequest()
//ApiLog is the SQS event details struct encoded as JSON document, sent to SQS, to be logged for each API handler executed.
type ApiLog struct {
......
......@@ -27,7 +27,25 @@ type TimeSeriesHeader struct {
type TimeSeries interface {
Write(StartTime time.Time, EndTime time.Time, data interface{}) error
Search(query Query, limit int64) (docs interface{}, totalCount int, err error)
// Search returns docs indexed on OpenSearch document ID which cat be used in Get(id)
// The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
// So you can safely type assert e.g.
// type myType struct {...}
// ts := search.TimeSeries(..., myType{})
// docs,totalCount,err := ts.Search(...)
// if err == nil {
// for id,docValue := range docs {
// doc := docValue.(myType)
// ...
// }
// }
Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error)
// Get takes the id returned in Search()
// The id is uuid assigned by OpenSearch when documents are added with Write().
// The document value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
Get(id string) (interface{}, error)
}
type timeSeries struct {
......@@ -41,9 +59,10 @@ type timeSeries struct {
createdDates map[string]bool
searchResponseBodyType reflect.Type
getResponseBodyType reflect.Type
}
//purpose:
// TimeSeries purpose:
// create a time series to write e.g. api api_logs
// parameters:
// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-api_logs"
......@@ -114,6 +133,18 @@ func (w *writer) TimeSeries(name string, tmpl interface{}) (TimeSeries, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to make search response type for time-series")
}
// define get response type
// similar to GetResponseBody
ts.getResponseBodyType, err = reflection.CloneType(
reflect.TypeOf(GetResponseBody{}),
map[string]reflect.Type{
"._source": ts.dataType,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to make get response type for time-series")
}
w.timeSeriesByName[name] = ts
return ts, nil
}
......@@ -360,7 +391,7 @@ type IndexSettings struct {
//Search
//Return:
// docs will be a slice of the TimeSeries data type
func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalCount int, err error) {
func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error) {
if ts == nil {
return nil, 0, errors.Errorf("time series == nil")
}
......@@ -425,10 +456,66 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
return nil, 0, nil //no matches
}
items, err := reflection.Get(resBodyPtrValue, ".hits.hits[]._source")
hits, err := reflection.Get(resBodyPtrValue, ".hits.hits[]")
if err != nil {
err = errors.Wrapf(err, "cannot get search response documents")
return
}
return items.Interface(), hitsTotalValue.Interface().(int), nil
docs = map[string]interface{}{}
for i := 0; i < hits.Len(); i++ {
hit := hits.Index(i)
index := hit.Field(0).Interface().(string) // HitDoc.Index
id := hit.Field(2).Interface().(string) // HitDoc.ID
docs[index+"/"+id] = hit.Field(4).Interface() // HitDoc.Source
}
return docs, hitsTotalValue.Interface().(int), nil
}
func (ds *timeSeries) Get(id string) (doc interface{}, err error) {
if ds == nil {
return nil, errors.Errorf("document store == nil")
}
parts := strings.SplitN(id, "/", 2)
get := opensearchapi.GetRequest{
Index: parts[0],
DocumentType: "_doc",
DocumentID: parts[1],
}
getResponse, err := get.Do(context.Background(), ds.w.client)
if err != nil {
err = errors.Wrapf(err, "failed to get document")
return
}
switch getResponse.StatusCode {
case http.StatusOK:
default:
resBody, _ := ioutil.ReadAll(getResponse.Body)
err = errors.Errorf("Get failed with HTTP status %v: %s", getResponse.StatusCode, string(resBody))
return
}
resBodyPtrValue := reflect.New(ds.getResponseBodyType)
if err = json.NewDecoder(getResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
err = errors.Wrapf(err, "cannot decode get response body")
return
}
foundVar, err := reflection.Get(resBodyPtrValue, ".found")
if err != nil {
err = errors.Wrapf(err, "cannot get found value")
return
}
if found, ok := foundVar.Interface().(bool); !ok || !found {
return nil, nil //not found
}
//found
source, err := reflection.Get(resBodyPtrValue, "._source")
if err != nil {
err = errors.Wrapf(err, "cannot get document from get response")
return
}
return source.Interface(), nil
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment