diff --git a/api_logs/api-logs.go b/api_logs/api-logs.go
index 2bb51c29b06df8a62acd54ee7bc8b95dc97f5d7e..0c797b7e98d2c89ee8f56a0e3bf46433b06173c5 100644
--- a/api_logs/api-logs.go
+++ b/api_logs/api-logs.go
@@ -2,7 +2,6 @@ package api_logs
import (
"net/url"
- "os"
"strconv"
"strings"
"time"
@@ -10,40 +9,7 @@ import (
"github.com/aws/aws-lambda-go/events"
)
-var (
- MaxReqBodyLength int = 1024
- MaxResBodyLength int = 1024
-)
-
-func init() {
- if s := os.Getenv("API_LOGS_MAX_REQ_BODY_LENGTH"); s != "" {
- if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 {
- MaxReqBodyLength = int(i64)
- }
- }
- if s := os.Getenv("API_LOGS_MAX_RES_BODY_LENGTH"); s != "" {
- if i64, err := strconv.ParseInt(s, 10, 64); err == nil && i64 >= 0 {
- MaxResBodyLength = int(i64)
- }
- }
-}
-
-//var producer queues.Producer
-
-//func Init(p queues.Producer) {
-// producer = p
-//}
-
-//Call this at the end of an API request handler to capture the req/res as well as all actions taken during the processing
-//(note: action list is only reset when this is called - so must be called after each handler, else action list has to be reset at the start)
-func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) error {
- //if producer == nil {
- // return errors.Errorf("api_logs queue producer not set")
- //}
-
- //todo: filter out some noisy (method+path)
- //logs.Debugf("claim: %+v", claim)
-
+func GenerateIncomingAPILog(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) ApiLog {
endTime := time.Now()
var authType string
@@ -93,57 +59,24 @@ func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[stri
Headers: req.Headers,
QueryParameters: req.QueryStringParameters,
BodySize: len(req.Body),
- //see below: Body: req.Body,
+ Body: req.Body,
},
Response: ApiLogResponse{
Headers: res.Headers,
BodySize: len(res.Body),
- //see below: Body: res.Body,
+ Body: res.Body,
},
}
- if apiLog.Request.BodySize > MaxReqBodyLength {
- apiLog.Request.Body = req.Body[:MaxReqBodyLength] + "..."
- } else {
- apiLog.Request.Body = req.Body
- }
- if apiLog.Response.BodySize > MaxResBodyLength {
- apiLog.Response.Body = res.Body[:MaxResBodyLength] + "..."
- } else {
- apiLog.Response.Body = res.Body
- }
//also copy multi-value query parameters to the log as CSV array values
for n, as := range req.MultiValueQueryStringParameters {
apiLog.Request.QueryParameters[n] = "[" + strings.Join(as, ",") + "]"
}
- //todo: filter out excessive req/res body content per (method+path)
- //todo: also need to do for all actions...
- // if apiLog.Method == http.MethodGet {
- // apiLog.Response.Body = "<not logged>"
- // }
-
- //todo: filter out sensitive values (e.g. OTP)
- //if _, err := producer.NewEvent("API_LOGS").
- // Type("api-log").
- // RequestID(apiLog.RequestID).
- // Send(apiLog); err != nil {
- // return errors.Wrapf(err, "failed to send api-log")
- //}
- return nil
-} //LogIncomingAPIRequest()
-
-//Call LogOutgoingAPIRequest() after calling an API end-point as part of a handler,
-//to capture the details
-//and add it to the current handler log story for reporting/metrics
-func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, responseBody string, responseCode int) error {
- //if producer == nil {
- // return errors.Errorf("api_logs queue producer not set")
- //}
-
- //todo: filter out some noisy (method+path)
- //logs.Debugf("claim: %+v", claim)
+ return apiLog
+}
+func GenerateOutgoingAPILog(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, requestHeaders map[string]string, responseBody string, responseCode int) ApiLog {
endTime := time.Now()
userID, _ := claim["UserID"].(int64)
username, _ := claim["Username"].(string)
@@ -170,38 +103,20 @@ func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[stri
Username: username,
AccountID: accountID,
Request: ApiLogRequest{
- //Headers: req.Headers,
+ Headers: requestHeaders,
QueryParameters: params,
BodySize: len(requestBody),
- //See below: Body: requestBody,
+ Body: requestBody,
},
Response: ApiLogResponse{
- //Headers: res.Headers,
+ Headers: requestHeaders,
BodySize: len(responseBody),
- //See below: Body: responseBody,
+ Body: responseBody,
},
}
- if apiLog.Request.BodySize > MaxReqBodyLength {
- apiLog.Request.Body = requestBody[:MaxReqBodyLength] + "..."
- } else {
- apiLog.Request.Body = requestBody
- }
- if apiLog.Response.BodySize > MaxResBodyLength {
- apiLog.Response.Body = responseBody[:MaxResBodyLength] + "..."
- } else {
- apiLog.Response.Body = responseBody
- }
-
- //todo: filter out sensitive values (e.g. OTP)
- //if _, err := producer.NewEvent("API_LOGS").
- // Type("api-log").
- // RequestID(apiLog.RequestID).
- // Send(apiLog); err != nil {
- // return errors.Wrapf(err, "failed to send api-log")
- //}
- return nil
-} //LogOutgoingAPIRequest()
+ return apiLog
+}
//ApiLog is the SQS event details struct encoded as JSON document, sent to SQS, to be logged for each API handler executed.
type ApiLog struct {
diff --git a/search/time_series.go b/search/time_series.go
index e9f87dd5fc3a8bbbf45549a74b27e233ad2b9ae9..837878f068f3a9286f052e4a61652728c0cc4325 100644
--- a/search/time_series.go
+++ b/search/time_series.go
@@ -27,7 +27,25 @@ type TimeSeriesHeader struct {
type TimeSeries interface {
Write(StartTime time.Time, EndTime time.Time, data interface{}) error
- Search(query Query, limit int64) (docs interface{}, totalCount int, err error)
+
+ // Search returns docs indexed on OpenSearch document ID which cat be used in Get(id)
+ // The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
+ // So you can safely type assert e.g.
+ // type myType struct {...}
+ // ts := search.TimeSeries(..., myType{})
+ // docs,totalCount,err := ts.Search(...)
+ // if err == nil {
+ // for id,docValue := range docs {
+ // doc := docValue.(myType)
+ // ...
+ // }
+ // }
+ Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error)
+
+ // Get takes the id returned in Search()
+ // The id is uuid assigned by OpenSearch when documents are added with Write().
+ // The document value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
+ Get(id string) (interface{}, error)
}
type timeSeries struct {
@@ -41,11 +59,12 @@ type timeSeries struct {
createdDates map[string]bool
searchResponseBodyType reflect.Type
+ getResponseBodyType reflect.Type
}
-//purpose:
-// create a time series to write e.g. api api_logs
-//parameters:
+// TimeSeries purpose:
+// create a time series to write e.g. api api_logs
+// parameters:
// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-api_logs"
// the actual indices in openSearch will be called "<indexName>-<ccyymmdd>" e.g. "uafrica-v3-api-api_logs-20210102"
// tmpl must be your log data struct consisting of public fields as:
@@ -114,6 +133,18 @@ func (w *writer) TimeSeries(name string, tmpl interface{}) (TimeSeries, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to make search response type for time-series")
}
+
+ // define get response type
+ // similar to GetResponseBody
+ ts.getResponseBodyType, err = reflection.CloneType(
+ reflect.TypeOf(GetResponseBody{}),
+ map[string]reflect.Type{
+ "._source": ts.dataType,
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to make get response type for time-series")
+ }
+
w.timeSeriesByName[name] = ts
return ts, nil
}
@@ -360,7 +391,7 @@ type IndexSettings struct {
//Search
//Return:
// docs will be a slice of the TimeSeries data type
-func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalCount int, err error) {
+func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error) {
if ts == nil {
return nil, 0, errors.Errorf("time series == nil")
}
@@ -425,10 +456,66 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
return nil, 0, nil //no matches
}
- items, err := reflection.Get(resBodyPtrValue, ".hits.hits[]._source")
+ hits, err := reflection.Get(resBodyPtrValue, ".hits.hits[]")
if err != nil {
err = errors.Wrapf(err, "cannot get search response documents")
return
}
- return items.Interface(), hitsTotalValue.Interface().(int), nil
+
+ docs = map[string]interface{}{}
+ for i := 0; i < hits.Len(); i++ {
+ hit := hits.Index(i)
+ index := hit.Field(0).Interface().(string) // HitDoc.Index
+ id := hit.Field(2).Interface().(string) // HitDoc.ID
+ docs[index+"/"+id] = hit.Field(4).Interface() // HitDoc.Source
+ }
+ return docs, hitsTotalValue.Interface().(int), nil
+}
+
+func (ds *timeSeries) Get(id string) (doc interface{}, err error) {
+ if ds == nil {
+ return nil, errors.Errorf("document store == nil")
+ }
+ parts := strings.SplitN(id, "/", 2)
+ get := opensearchapi.GetRequest{
+ Index: parts[0],
+ DocumentType: "_doc",
+ DocumentID: parts[1],
+ }
+ getResponse, err := get.Do(context.Background(), ds.w.client)
+ if err != nil {
+ err = errors.Wrapf(err, "failed to get document")
+ return
+ }
+
+ switch getResponse.StatusCode {
+ case http.StatusOK:
+ default:
+ resBody, _ := ioutil.ReadAll(getResponse.Body)
+ err = errors.Errorf("Get failed with HTTP status %v: %s", getResponse.StatusCode, string(resBody))
+ return
+ }
+
+ resBodyPtrValue := reflect.New(ds.getResponseBodyType)
+ if err = json.NewDecoder(getResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
+ err = errors.Wrapf(err, "cannot decode get response body")
+ return
+ }
+
+ foundVar, err := reflection.Get(resBodyPtrValue, ".found")
+ if err != nil {
+ err = errors.Wrapf(err, "cannot get found value")
+ return
+ }
+ if found, ok := foundVar.Interface().(bool); !ok || !found {
+ return nil, nil //not found
+ }
+
+ //found
+ source, err := reflection.Get(resBodyPtrValue, "._source")
+ if err != nil {
+ err = errors.Wrapf(err, "cannot get document from get response")
+ return
+ }
+ return source.Interface(), nil
}