From 8b95a28568eb5fbe0a842442cdd04bb1d825fe59 Mon Sep 17 00:00:00 2001
From: Jan Semmelink <jan@uafrica.com>
Date: Mon, 29 Nov 2021 11:17:25 +0200
Subject: [PATCH] Changed Search response to include document ids
---
search/README.md | 24 +++++++++++++++++++++++-
search/search_test.go | 18 ++++++++++--------
search/time_series.go | 37 +++++++++++++++++++++++++++++++++----
3 files changed, 66 insertions(+), 13 deletions(-)
diff --git a/search/README.md b/search/README.md
index b956a80..8e1b934 100644
--- a/search/README.md
+++ b/search/README.md
@@ -15,4 +15,26 @@ We use a document store and search to provide quick text searches from the API.
When a user is looking for an order, the API provides an end-point to search orders e.g. for "lcd screen", then the API does an OpenSearch query in the orders index, get N results and then read those orders from the orders table in the database (not OpenSearch) and return those results to the user.
-We therefore use OpenSearch only for searching and returning a list of document ids, then read the documents from the database. A document is typically an "order" but also anything else that we need to do free text searches on.
\ No newline at end of file
+We therefore use OpenSearch only for searching and returning a list of document ids, then read the documents from the database. A document is typically an "order" but also anything else that we need to do free text searches on.
+
+## Testing
+The dev sub-directory contains a docker-compose.yml that runs OpenSearch loccally for test programs.
+Start it with:
+```
+ cd dev
+ docker-compose up -d
+```
+Then run the go test programs in this directory...
+E.g.:
+```go test -v --run TestLocalWriter```
+
+To work with this local instance from the command line:
+```curl --insecure -uadmin:admin "https://localhost:9200/_cat/indices"```
+
+If the test fail with index mapping error, you can delete the index before running the test, with the following command. It often happens when the code that generate the mapping changed and the existing index is incompatible with the new mapping:
+```
+curl --insecure -uadmin:admin -XDELETE "https://localhost:9200/go-utils-search-docs-test"
+```
+
+Some of the test programs also refer to the cloud instance created manually in V3, e.g. search_test.go TestDevWriter(). That can be updated or deleted as required.
+
diff --git a/search/search_test.go b/search/search_test.go
index 8a03042..f3dd0be 100644
--- a/search/search_test.go
+++ b/search/search_test.go
@@ -42,7 +42,7 @@ func test(t *testing.T, c search.Config) {
//write N records
methods := []string{"GET", "POST", "GET", "PATCH", "GET", "GET", "DELETE", "GET", "GET"} //more gets than others
paths := []string{"/users", "/orders", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates"}
- N := 100
+ N := 1
testTime := time.Now().Add(-time.Hour * time.Duration(N))
for i := 0; i < N; i++ {
testTime = testTime.Add(time.Duration(float64(rand.Intn(100)) / 60.0 * float64(time.Hour)))
@@ -74,17 +74,19 @@ func test(t *testing.T, c search.Config) {
},
}
- docs, totalCount, err := ts.Search(query, 10)
+ docsByIDMap, totalCount, err := ts.Search(query, 10)
if err != nil {
t.Errorf("failed to search: %+v", err)
} else {
- if docsSlice, ok := docs.([]testStruct); ok {
- t.Logf("search result total_count:%d with %d docs", totalCount, len(docsSlice))
- if len(docsSlice) > 10 {
- t.Errorf("got %d docs > max 10", len(docsSlice))
+ t.Logf("search result total_count:%d with %d docs", totalCount, len(docsByIDMap))
+ if len(docsByIDMap) > 10 {
+ t.Errorf("got %d docs > max 10", len(docsByIDMap))
+ }
+ for id, doc := range docsByIDMap {
+ t.Logf("id=%s doc=(%T)%+v", id, doc, doc)
+ if _, ok := doc.(testStruct); !ok {
+ t.Errorf("docs %T is not testStruct!", docsByIDMap)
}
- } else {
- t.Errorf("docs %T is not []testStruct!", docs)
}
}
diff --git a/search/time_series.go b/search/time_series.go
index 1b39cb9..68f62e7 100644
--- a/search/time_series.go
+++ b/search/time_series.go
@@ -27,7 +27,25 @@ type TimeSeriesHeader struct {
type TimeSeries interface {
Write(StartTime time.Time, EndTime time.Time, data interface{}) error
- Search(query Query, limit int64) (docs interface{}, totalCount int, err error)
+
+ //Search() returns docs indexed on OpenSearch document ID which cat be used in Get(id)
+ //The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
+ //So you can safely type assert e.g.
+ // type myType struct {...}
+ // ts := search.TimeSeries(..., myType{})
+ // docs,totalCount,err := ts.Search(...)
+ // if err == nil {
+ // for id,docValue := range docs {
+ // doc := docValue.(myType)
+ // ...
+ // }
+ // }
+ Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error)
+
+ //Get() takes the id returned in Search()
+ //The id is uuid assigned by OpenSearch when documents are added with Write().
+ //The document value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
+ Get(id string) (interface{}, error)
}
type timeSeries struct {
@@ -360,7 +378,7 @@ type IndexSettings struct {
//Search
//Return:
// docs will be a slice of the TimeSeries data type
-func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalCount int, err error) {
+func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error) {
if ts == nil {
return nil, 0, errors.Errorf("time series == nil")
}
@@ -425,10 +443,21 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
return nil, 0, nil //no matches
}
- items, err := reflection.Get(resBodyPtrValue, ".hits.hits[]._source")
+ hits, err := reflection.Get(resBodyPtrValue, ".hits.hits[]")
if err != nil {
err = errors.Wrapf(err, "cannot get search response documents")
return
}
- return items.Interface(), hitsTotalValue.Interface().(int), nil
+
+ docs = map[string]interface{}{}
+ for i := 0; i < hits.Len(); i++ {
+ hit := hits.Index(i)
+ id := hit.Field(2).Interface().(string) //HitDoc.ID
+ docs[id] = hit.Field(4).Interface() //HitDoc.Source
+ }
+ return docs, hitsTotalValue.Interface().(int), nil
+}
+
+func (ts *timeSeries) Get(id string) (interface{}, error) {
+ return nil, errors.Errorf("NYI")
}
--
GitLab