Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
bobgroup-go-utils
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Bob Public Utils
bobgroup-go-utils
Commits
892d7326
Commit
892d7326
authored
3 years ago
by
Francé Wilke
Browse files
Options
Downloads
Patches
Plain Diff
Api logs - get by ID
parent
d4e877ef
No related branches found
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
api_logs/api-logs.go
+12
-97
12 additions, 97 deletions
api_logs/api-logs.go
search/time_series.go
+94
-7
94 additions, 7 deletions
search/time_series.go
with
106 additions
and
104 deletions
api_logs/api-logs.go
+
12
−
97
View file @
892d7326
...
...
@@ -2,7 +2,6 @@ package api_logs
import
(
"net/url"
"os"
"strconv"
"strings"
"time"
...
...
@@ -10,40 +9,7 @@ import (
"github.com/aws/aws-lambda-go/events"
)
var
(
MaxReqBodyLength
int
=
1024
MaxResBodyLength
int
=
1024
)
func
init
()
{
if
s
:=
os
.
Getenv
(
"API_LOGS_MAX_REQ_BODY_LENGTH"
);
s
!=
""
{
if
i64
,
err
:=
strconv
.
ParseInt
(
s
,
10
,
64
);
err
==
nil
&&
i64
>=
0
{
MaxReqBodyLength
=
int
(
i64
)
}
}
if
s
:=
os
.
Getenv
(
"API_LOGS_MAX_RES_BODY_LENGTH"
);
s
!=
""
{
if
i64
,
err
:=
strconv
.
ParseInt
(
s
,
10
,
64
);
err
==
nil
&&
i64
>=
0
{
MaxResBodyLength
=
int
(
i64
)
}
}
}
//var producer queues.Producer
//func Init(p queues.Producer) {
// producer = p
//}
//Call this at the end of an API request handler to capture the req/res as well as all actions taken during the processing
//(note: action list is only reset when this is called - so must be called after each handler, else action list has to be reset at the start)
func
LogIncomingAPIRequest
(
startTime
time
.
Time
,
requestID
string
,
claim
map
[
string
]
interface
{},
req
events
.
APIGatewayProxyRequest
,
res
events
.
APIGatewayProxyResponse
)
error
{
//if producer == nil {
// return errors.Errorf("api_logs queue producer not set")
//}
//todo: filter out some noisy (method+path)
//logs.Debugf("claim: %+v", claim)
func
GenerateIncomingAPILog
(
startTime
time
.
Time
,
requestID
string
,
claim
map
[
string
]
interface
{},
req
events
.
APIGatewayProxyRequest
,
res
events
.
APIGatewayProxyResponse
)
ApiLog
{
endTime
:=
time
.
Now
()
var
authType
string
...
...
@@ -93,57 +59,24 @@ func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[stri
Headers
:
req
.
Headers
,
QueryParameters
:
req
.
QueryStringParameters
,
BodySize
:
len
(
req
.
Body
),
//see below:
Body: req.Body,
Body
:
req
.
Body
,
},
Response
:
ApiLogResponse
{
Headers
:
res
.
Headers
,
BodySize
:
len
(
res
.
Body
),
//see below:
Body: res.Body,
Body
:
res
.
Body
,
},
}
if
apiLog
.
Request
.
BodySize
>
MaxReqBodyLength
{
apiLog
.
Request
.
Body
=
req
.
Body
[
:
MaxReqBodyLength
]
+
"..."
}
else
{
apiLog
.
Request
.
Body
=
req
.
Body
}
if
apiLog
.
Response
.
BodySize
>
MaxResBodyLength
{
apiLog
.
Response
.
Body
=
res
.
Body
[
:
MaxResBodyLength
]
+
"..."
}
else
{
apiLog
.
Response
.
Body
=
res
.
Body
}
//also copy multi-value query parameters to the log as CSV array values
for
n
,
as
:=
range
req
.
MultiValueQueryStringParameters
{
apiLog
.
Request
.
QueryParameters
[
n
]
=
"["
+
strings
.
Join
(
as
,
","
)
+
"]"
}
//todo: filter out excessive req/res body content per (method+path)
//todo: also need to do for all actions...
// if apiLog.Method == http.MethodGet {
// apiLog.Response.Body = "<not logged>"
// }
//todo: filter out sensitive values (e.g. OTP)
//if _, err := producer.NewEvent("API_LOGS").
// Type("api-log").
// RequestID(apiLog.RequestID).
// Send(apiLog); err != nil {
// return errors.Wrapf(err, "failed to send api-log")
//}
return
nil
}
//LogIncomingAPIRequest()
//Call LogOutgoingAPIRequest() after calling an API end-point as part of a handler,
//to capture the details
//and add it to the current handler log story for reporting/metrics
func
LogOutgoingAPIRequest
(
startTime
time
.
Time
,
requestID
string
,
claim
map
[
string
]
interface
{},
urlString
string
,
method
string
,
requestBody
string
,
responseBody
string
,
responseCode
int
)
error
{
//if producer == nil {
// return errors.Errorf("api_logs queue producer not set")
//}
//todo: filter out some noisy (method+path)
//logs.Debugf("claim: %+v", claim)
return
apiLog
}
func
GenerateOutgoingAPILog
(
startTime
time
.
Time
,
requestID
string
,
claim
map
[
string
]
interface
{},
urlString
string
,
method
string
,
requestBody
string
,
requestHeaders
map
[
string
]
string
,
responseBody
string
,
responseCode
int
)
ApiLog
{
endTime
:=
time
.
Now
()
userID
,
_
:=
claim
[
"UserID"
]
.
(
int64
)
username
,
_
:=
claim
[
"Username"
]
.
(
string
)
...
...
@@ -170,38 +103,20 @@ func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[stri
Username
:
username
,
AccountID
:
accountID
,
Request
:
ApiLogRequest
{
//
Headers: req
.
Headers,
Headers
:
req
uest
Headers
,
QueryParameters
:
params
,
BodySize
:
len
(
requestBody
),
//See below:
Body: requestBody,
Body
:
requestBody
,
},
Response
:
ApiLogResponse
{
//
Headers: re
s.
Headers,
Headers
:
re
quest
Headers
,
BodySize
:
len
(
responseBody
),
//See below:
Body: responseBody,
Body
:
responseBody
,
},
}
if
apiLog
.
Request
.
BodySize
>
MaxReqBodyLength
{
apiLog
.
Request
.
Body
=
requestBody
[
:
MaxReqBodyLength
]
+
"..."
}
else
{
apiLog
.
Request
.
Body
=
requestBody
return
apiLog
}
if
apiLog
.
Response
.
BodySize
>
MaxResBodyLength
{
apiLog
.
Response
.
Body
=
responseBody
[
:
MaxResBodyLength
]
+
"..."
}
else
{
apiLog
.
Response
.
Body
=
responseBody
}
//todo: filter out sensitive values (e.g. OTP)
//if _, err := producer.NewEvent("API_LOGS").
// Type("api-log").
// RequestID(apiLog.RequestID).
// Send(apiLog); err != nil {
// return errors.Wrapf(err, "failed to send api-log")
//}
return
nil
}
//LogOutgoingAPIRequest()
//ApiLog is the SQS event details struct encoded as JSON document, sent to SQS, to be logged for each API handler executed.
type
ApiLog
struct
{
...
...
This diff is collapsed.
Click to expand it.
search/time_series.go
+
94
−
7
View file @
892d7326
...
...
@@ -27,7 +27,25 @@ type TimeSeriesHeader struct {
type
TimeSeries
interface
{
Write
(
StartTime
time
.
Time
,
EndTime
time
.
Time
,
data
interface
{})
error
Search
(
query
Query
,
limit
int64
)
(
docs
interface
{},
totalCount
int
,
err
error
)
// Search returns docs indexed on OpenSearch document ID which cat be used in Get(id)
// The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
// So you can safely type assert e.g.
// type myType struct {...}
// ts := search.TimeSeries(..., myType{})
// docs,totalCount,err := ts.Search(...)
// if err == nil {
// for id,docValue := range docs {
// doc := docValue.(myType)
// ...
// }
// }
Search
(
query
Query
,
limit
int64
)
(
docs
map
[
string
]
interface
{},
totalCount
int
,
err
error
)
// Get takes the id returned in Search()
// The id is uuid assigned by OpenSearch when documents are added with Write().
// The document value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
Get
(
id
string
)
(
interface
{},
error
)
}
type
timeSeries
struct
{
...
...
@@ -41,9 +59,10 @@ type timeSeries struct {
createdDates
map
[
string
]
bool
searchResponseBodyType
reflect
.
Type
getResponseBodyType
reflect
.
Type
}
//purpose:
//
TimeSeries
purpose:
// create a time series to write e.g. api api_logs
// parameters:
// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-api_logs"
...
...
@@ -114,6 +133,18 @@ func (w *writer) TimeSeries(name string, tmpl interface{}) (TimeSeries, error) {
if
err
!=
nil
{
return
nil
,
errors
.
Wrapf
(
err
,
"failed to make search response type for time-series"
)
}
// define get response type
// similar to GetResponseBody
ts
.
getResponseBodyType
,
err
=
reflection
.
CloneType
(
reflect
.
TypeOf
(
GetResponseBody
{}),
map
[
string
]
reflect
.
Type
{
"._source"
:
ts
.
dataType
,
})
if
err
!=
nil
{
return
nil
,
errors
.
Wrapf
(
err
,
"failed to make get response type for time-series"
)
}
w
.
timeSeriesByName
[
name
]
=
ts
return
ts
,
nil
}
...
...
@@ -360,7 +391,7 @@ type IndexSettings struct {
//Search
//Return:
// docs will be a slice of the TimeSeries data type
func
(
ts
*
timeSeries
)
Search
(
query
Query
,
limit
int64
)
(
docs
interface
{},
totalCount
int
,
err
error
)
{
func
(
ts
*
timeSeries
)
Search
(
query
Query
,
limit
int64
)
(
docs
map
[
string
]
interface
{},
totalCount
int
,
err
error
)
{
if
ts
==
nil
{
return
nil
,
0
,
errors
.
Errorf
(
"time series == nil"
)
}
...
...
@@ -425,10 +456,66 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalC
return
nil
,
0
,
nil
//no matches
}
it
em
s
,
err
:=
reflection
.
Get
(
resBodyPtrValue
,
".hits.hits[]
._source
"
)
h
its
,
err
:=
reflection
.
Get
(
resBodyPtrValue
,
".hits.hits[]"
)
if
err
!=
nil
{
err
=
errors
.
Wrapf
(
err
,
"cannot get search response documents"
)
return
}
return
items
.
Interface
(),
hitsTotalValue
.
Interface
()
.
(
int
),
nil
docs
=
map
[
string
]
interface
{}{}
for
i
:=
0
;
i
<
hits
.
Len
();
i
++
{
hit
:=
hits
.
Index
(
i
)
index
:=
hit
.
Field
(
0
)
.
Interface
()
.
(
string
)
// HitDoc.Index
id
:=
hit
.
Field
(
2
)
.
Interface
()
.
(
string
)
// HitDoc.ID
docs
[
index
+
"/"
+
id
]
=
hit
.
Field
(
4
)
.
Interface
()
// HitDoc.Source
}
return
docs
,
hitsTotalValue
.
Interface
()
.
(
int
),
nil
}
func
(
ds
*
timeSeries
)
Get
(
id
string
)
(
doc
interface
{},
err
error
)
{
if
ds
==
nil
{
return
nil
,
errors
.
Errorf
(
"document store == nil"
)
}
parts
:=
strings
.
SplitN
(
id
,
"/"
,
2
)
get
:=
opensearchapi
.
GetRequest
{
Index
:
parts
[
0
],
DocumentType
:
"_doc"
,
DocumentID
:
parts
[
1
],
}
getResponse
,
err
:=
get
.
Do
(
context
.
Background
(),
ds
.
w
.
client
)
if
err
!=
nil
{
err
=
errors
.
Wrapf
(
err
,
"failed to get document"
)
return
}
switch
getResponse
.
StatusCode
{
case
http
.
StatusOK
:
default
:
resBody
,
_
:=
ioutil
.
ReadAll
(
getResponse
.
Body
)
err
=
errors
.
Errorf
(
"Get failed with HTTP status %v: %s"
,
getResponse
.
StatusCode
,
string
(
resBody
))
return
}
resBodyPtrValue
:=
reflect
.
New
(
ds
.
getResponseBodyType
)
if
err
=
json
.
NewDecoder
(
getResponse
.
Body
)
.
Decode
(
resBodyPtrValue
.
Interface
());
err
!=
nil
{
err
=
errors
.
Wrapf
(
err
,
"cannot decode get response body"
)
return
}
foundVar
,
err
:=
reflection
.
Get
(
resBodyPtrValue
,
".found"
)
if
err
!=
nil
{
err
=
errors
.
Wrapf
(
err
,
"cannot get found value"
)
return
}
if
found
,
ok
:=
foundVar
.
Interface
()
.
(
bool
);
!
ok
||
!
found
{
return
nil
,
nil
//not found
}
//found
source
,
err
:=
reflection
.
Get
(
resBodyPtrValue
,
"._source"
)
if
err
!=
nil
{
err
=
errors
.
Wrapf
(
err
,
"cannot get document from get response"
)
return
}
return
source
.
Interface
(),
nil
}
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment