From fd8b9b2062a8d904462093ad4331915687e81eab Mon Sep 17 00:00:00 2001
From: Jan Semmelink <jan@uafrica.com>
Date: Mon, 15 Nov 2021 10:28:28 +0200
Subject: [PATCH] Update structs for search on api-logs
---
logs/api-logs.go | 20 ++++++++++----------
search/document_store.go | 1 +
search/search_test.go | 4 ++++
search/time_series.go | 10 ++++++++--
4 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/logs/api-logs.go b/logs/api-logs.go
index 01900b2..abbad45 100644
--- a/logs/api-logs.go
+++ b/logs/api-logs.go
@@ -108,17 +108,17 @@ type ApiLog struct {
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
DurMs int64 `json:"duration_ms"` //duration in milliseconds
- Method string `json:"method" search:"keyword"`
+ Method string `json:"method"`
Address string `json:"address"` //server address for incoming and outgoing
- Path string `json:"path" search:"keyword"`
- ResponseCode int `json:"response_code" search:"keyword"`
- RequestID string `json:"request_id" search:"keyword"`
- InitialAuthUsername string `json:"initial_auth_username,omitempty" search:"keyword"`
- InitialAuthType string `json:"initial_auth_type,omitempty" search:"keyword"`
- AccountID int64 `json:"account_id,omitempty" search:"keyword"`
- Username string `json:"username,omitempty" search:"keyword"`
- SourceIP string `json:"source_ip,omitempty" search:"keyword"` //only logged for incoming API
- UserAgent string `json:"user_agent,omitempty"` //only for incoming, indicate type of browser when UI
+ Path string `json:"path"`
+ ResponseCode int `json:"response_code"`
+ RequestID string `json:"request_id"`
+ InitialAuthUsername string `json:"initial_auth_username,omitempty"`
+ InitialAuthType string `json:"initial_auth_type,omitempty"`
+ AccountID int64 `json:"account_id,omitempty"`
+ Username string `json:"username,omitempty"`
+ SourceIP string `json:"source_ip,omitempty"` //only logged for incoming API
+ UserAgent string `json:"user_agent,omitempty"` //only for incoming, indicate type of browser when UI
RelevantID string `json:"relevant_id,omitempty"`
Request ApiLogRequest `json:"request"`
Response ApiLogResponse `json:"response"`
diff --git a/search/document_store.go b/search/document_store.go
index 24e5fbd..29cb32a 100644
--- a/search/document_store.go
+++ b/search/document_store.go
@@ -221,6 +221,7 @@ func (ds *documentStore) Search(query Query, limit int64) (ids []string, totalCo
resBodyPtrValue := reflect.New(ds.searchResponseBodyType)
// if err = json.NewDecoder(searchResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
if err = json.Unmarshal(bodyData, resBodyPtrValue.Interface()); err != nil {
+ logger.Errorf("search response body: %s", string(bodyData))
err = errors.Wrapf(err, "cannot decode search response body")
return
}
diff --git a/search/search_test.go b/search/search_test.go
index 2dec817..8a03042 100644
--- a/search/search_test.go
+++ b/search/search_test.go
@@ -134,3 +134,7 @@ func TestOlderThan(t *testing.T) {
t1 = t1.Add(-time.Hour * 24 * time.Duration(olderThanDays))
t.Logf("Threshold = %s", t1)
}
+
+func TestTime(t *testing.T) {
+ t.Logf("Time: %s", time.Now().Format("2006-01-02T15:04:05Z07:00"))
+}
diff --git a/search/time_series.go b/search/time_series.go
index f867470..423d22c 100644
--- a/search/time_series.go
+++ b/search/time_series.go
@@ -16,7 +16,7 @@ import (
"gitlab.com/uafrica/go-utils/reflection"
)
-const TimeFormat = "2006-01-02T15:04:05+07:00"
+const TimeFormat = "2006-01-02T15:04:05Z07:00"
//embed this into your log struct
type TimeSeriesHeader struct {
@@ -384,6 +384,7 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
Query: query,
}
jsonBody, _ := json.Marshal(body)
+ logger.Debugf("Search: %s", string(jsonBody))
search := opensearchapi.SearchRequest{
Index: []string{ts.name + "-*"},
Body: bytes.NewReader(jsonBody),
@@ -403,8 +404,13 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
return
}
+ bodyData, _ := ioutil.ReadAll(searchResponse.Body)
+ logger.Debugf("Response Body: %s", string(bodyData))
+
resBodyPtrValue := reflect.New(ts.searchResponseBodyType)
- if err = json.NewDecoder(searchResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
+ //if err = json.NewDecoder(searchResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
+ if err = json.Unmarshal(bodyData, resBodyPtrValue.Interface()); err != nil {
+ logger.Errorf("search response body: %s", string(bodyData))
err = errors.Wrapf(err, "cannot decode search response body")
return
}
--
GitLab