From 7d54e7f76c62ebcf8687274b73439adaac141a63 Mon Sep 17 00:00:00 2001
From: Jan Semmelink <jan@uafrica.com>
Date: Mon, 15 Nov 2021 13:25:17 +0200
Subject: [PATCH] Added search to logs actions
---
logs/action.go | 30 ++++++++++++++++++++++++++++++
logs/config.go | 11 ++++++-----
search/document_store.go | 8 ++++++++
search/time_series.go | 7 +++++++
4 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/logs/action.go b/logs/action.go
index 423fd3c..23f1a61 100644
--- a/logs/action.go
+++ b/logs/action.go
@@ -111,6 +111,22 @@ func LogSQSSent(startTime time.Time, queueName string, messageType string, reque
actionListMutex.Unlock()
}
+func LogSearch(startTime time.Time, index string, query string) {
+ endTime := time.Now()
+ actionListMutex.Lock()
+ actionList = append(actionList, ActionLog{
+ StartTime: startTime,
+ EndTime: endTime,
+ DurMs: endTime.Sub(startTime).Milliseconds(),
+ Type: ActionTypeSearch,
+ Search: &SearchLog{
+ Index: index,
+ Query: query,
+ },
+ })
+ actionListMutex.Unlock()
+}
+
//Call LogSleep() after doing time.Sleep()
//to capture the details
//and add it to the current handler log story for reporting/metrics
@@ -139,6 +155,7 @@ type ActionLog struct {
ApiCall *ApiCallLog `json:"api_call,omitempty"`
SQSSent *SQSSentLog `json:"sqs_sent,omitempty"`
SQLQuery *SQLQueryLog `json:"sql_query,omitempty"`
+ Search *SearchLog `json:"search"`
}
func (action ActionLog) Relative(relTime time.Time) RelativeActionLog {
@@ -170,6 +187,7 @@ var ActionTypeList = []ActionType{
ActionTypeApiCall,
ActionTypeSqsSent,
ActionTypeSqlQuery,
+ ActionTypeSearch,
ActionTypeSleep,
}
@@ -178,6 +196,7 @@ const (
ActionTypeApiCall ActionType = "api-call"
ActionTypeSqsSent ActionType = "sqs-sent"
ActionTypeSqlQuery ActionType = "sql-query"
+ ActionTypeSearch ActionType = "search"
ActionTypeSleep ActionType = "sleep"
)
@@ -210,6 +229,11 @@ type SQLQueryLog struct {
Error string `json:"error,omitempty"`
}
+type SearchLog struct {
+ Index string `json:"index"`
+ Query string `json:"query"`
+}
+
//compile the relative action list that took place during this handler
//copy then reset actionList for the next handler
//we copy it with relation to this API's start..end time, rather than full timestamps, which are hard to read in the list
@@ -263,6 +287,12 @@ func relativeActionList(startTime, endTime time.Time) []RelativeActionLog {
action.ApiCall.Response.Body = action.ApiCall.Response.Body[:cfg.ActionsMaxAPIResBodyLength]
}
}
+ case ActionTypeSearch:
+ if action.Search != nil {
+ if len(action.Search.Query) > int(cfg.ActionsMaxSearchQueryLength) {
+ action.Search.Query = action.Search.Query[:cfg.ActionsMaxSearchQueryLength]
+ }
+ }
}
//make relative and append to the list
diff --git a/logs/config.go b/logs/config.go
index f05411e..f969464 100644
--- a/logs/config.go
+++ b/logs/config.go
@@ -10,11 +10,12 @@ import (
)
type Config struct {
- ActionsKeep bool `json:"actions_keep" doc:"Set true to keep list of actions in logs"`
- ActionsMaxSQLLength int64 `json:"actions_max_sql_length" doc:"Set length of SQL query to keep in action list (default 0 = delete)"`
- ActionsMaxSQSReqBodyLength int64 `json:"actions_max_sqs_req_body_length" doc:"Set length of SQS Request body to keep in action list (default 0 = delete)"`
- ActionsMaxAPIReqBodyLength int64 `json:"actions_max_api_req_body_length" doc:"Set length of API Request body to keep in action list (default 0 = delete)"`
- ActionsMaxAPIResBodyLength int64 `json:"actions_max_api_res_body_length" doc:"Set length of API Response body to keep in action list (default 0 = delete)"`
+ ActionsKeep bool `json:"actions_keep" doc:"Set true to keep list of actions in logs"`
+ ActionsMaxSQLLength int64 `json:"actions_max_sql_length" doc:"Set length of SQL query to keep in action list (default 0 = delete)"`
+ ActionsMaxSQSReqBodyLength int64 `json:"actions_max_sqs_req_body_length" doc:"Set length of SQS Request body to keep in action list (default 0 = delete)"`
+ ActionsMaxAPIReqBodyLength int64 `json:"actions_max_api_req_body_length" doc:"Set length of API Request body to keep in action list (default 0 = delete)"`
+ ActionsMaxAPIResBodyLength int64 `json:"actions_max_api_res_body_length" doc:"Set length of API Response body to keep in action list (default 0 = delete)"`
+ ActionsMaxSearchQueryLength int64 `json:"actions_max_search_query_length" doc:"Set length of search query to keep in action list (default 0 = delete)"`
}
const configPrefix = "LOGS"
diff --git a/search/document_store.go b/search/document_store.go
index 29cb32a..2c040d2 100644
--- a/search/document_store.go
+++ b/search/document_store.go
@@ -8,10 +8,12 @@ import (
"net/http"
"reflect"
"strings"
+ "time"
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/reflection"
)
@@ -195,12 +197,18 @@ func (ds *documentStore) Search(query Query, limit int64) (ids []string, totalCo
Size: limit,
Query: query,
}
+
jsonBody, _ := json.Marshal(body)
search := opensearchapi.SearchRequest{
Index: []string{ds.name},
Body: bytes.NewReader(jsonBody),
}
+ startTime := time.Now()
+ defer func() {
+ logs.LogSearch(startTime, ds.name, string(jsonBody))
+ }()
+
searchResponse, err := search.Do(context.Background(), ds.w.client)
if err != nil {
err = errors.Wrapf(err, "failed to search documents")
diff --git a/search/time_series.go b/search/time_series.go
index 423d22c..0fddffa 100644
--- a/search/time_series.go
+++ b/search/time_series.go
@@ -13,6 +13,7 @@ import (
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/reflection"
)
@@ -383,6 +384,7 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
Size: limit,
Query: query,
}
+
jsonBody, _ := json.Marshal(body)
logger.Debugf("Search: %s", string(jsonBody))
search := opensearchapi.SearchRequest{
@@ -390,6 +392,11 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
Body: bytes.NewReader(jsonBody),
}
+ startTime := time.Now()
+ defer func() {
+ logs.LogSearch(startTime, ts.name+"-*", string(jsonBody))
+ }()
+
searchResponse, err := search.Do(context.Background(), ts.w.client)
if err != nil {
err = errors.Wrapf(err, "failed to search documents")
--
GitLab