diff --git a/go.mod b/go.mod
index 68fb4cecf343267af1aa5f195d8b89a317909125..b976fa9abe37d4142fecf9f60790fa07245ebaee 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils
-go 1.19
+go 1.21
require (
github.com/MindscapeHQ/raygun4go v1.1.1
diff --git a/handler_utils/request.go b/handler_utils/request.go
index 8d251bf0739e0abb77c65c5732390b2005695ba4..ac13d59ee7dacbb6d98c2110c7f13d9bc91a5240 100644
--- a/handler_utils/request.go
+++ b/handler_utils/request.go
@@ -15,7 +15,7 @@ import (
"github.com/aws/aws-lambda-go/lambdacontext"
)
-const HTTPXRequestIDHeaderValue = "X-Request-ID"
+const HTTPXRequestIDHeaderKey = "X-Request-ID"
func RequestIDFromLambdaContext(ctx context.Context) *string {
// Get request ID from context
@@ -39,14 +39,14 @@ func RequestIDFromHeaders(headers map[string]string, requestIDHeaderKey string)
func AddRequestIDToHeaders(requestID *string, responseHeaders map[string]string, requestIDHeaderKey string, requestHeaders map[string]string) {
if requestID != nil && responseHeaders != nil {
responseHeaders[requestIDHeaderKey] = *requestID
- responseHeaders[HTTPXRequestIDHeaderValue] = *requestID
+ responseHeaders[HTTPXRequestIDHeaderKey] = *requestID
}
// Add the HTTP X-Request-ID request header to the response headers: https://http.dev/x-request-id
for key, val := range requestHeaders {
// Don't be case-sensitive
- if strings.ToLower(key) == strings.ToLower(HTTPXRequestIDHeaderValue) {
- responseHeaders[key] = val
+ if strings.ToLower(key) == strings.ToLower(HTTPXRequestIDHeaderKey) {
+ responseHeaders[HTTPXRequestIDHeaderKey] = val
break
}
}
diff --git a/logs/database_logs.go b/logs/database_logs.go
index cec7db69f657f47c5001a0b07a088d857ccf62c5..3edece0ccb41b957c1c3f69e31a46700d9f4a4ad 100644
--- a/logs/database_logs.go
+++ b/logs/database_logs.go
@@ -31,7 +31,8 @@ var ignoredTableInserts = []string{
}
type QueryHook struct {
- Debug bool
+ IgnoredTableInserts []string
+ Debug bool
}
func (d QueryHook) BeforeQuery(ctx context.Context, _ *bun.QueryEvent) context.Context {
@@ -47,8 +48,9 @@ func (d QueryHook) AfterQuery(_ context.Context, event *bun.QueryEvent) {
shouldLogQuery := !strings.Contains(sqlQuery, "api_key")
// Don't log queries for certain tables
+ ignoredTableInsertsCombined := append(ignoredTableInserts, d.IgnoredTableInserts...)
tableName := TableNameForQuery(event)
- if lo.Contains(ignoredTableInserts, tableName) {
+ if lo.Contains(ignoredTableInsertsCombined, tableName) {
shouldLogQuery = false
}
diff --git a/logs/logs.go b/logs/logs.go
index 99aa65e5f6179e2d8643a9b6797d3ac9a7d515c2..0cb605d113ea7e900fb908ffe85549b82492d434 100644
--- a/logs/logs.go
+++ b/logs/logs.go
@@ -30,33 +30,32 @@ var build string
var raygunClient *raygun4go.Client
// Password filtering
-var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?",).*`)
+var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?").*`)
+var byteArrayRegex = regexp.MustCompile(`(?i:\\?"(?i:[\w]*)(?i:byte|data)(?i:[\w]*)\\?"\s*:\s*\[([\d\s,]+)*\])`)
func SanitiseLogs(logString string) string {
+ var isValidJsonString bool
+ isValidJsonString, logString = string_utils.PrettyJSON(logString)
+ if !isValidJsonString {
+ return logString
+ }
+
+ logString = MaskByteArraysInJsonString(logString)
logString = MaskPasswordsInJsonString(logString)
return logString
}
-// MaskPasswordsInJsonString takes a string and, if it is a JSON string, sanitises all the password. In order for the
-// regex to work correctly we need to prettify the JSON, so the function always returns a formatted JSON string.
+// MaskPasswordsInJsonString takes a string and sanitises all the instances of fields named password.
+// E.g. "{"password": "xyz123"}" will become "{"password": "***"}"
func MaskPasswordsInJsonString(jsonString string) string {
- var isValidJsonString bool
- isValidJsonString, jsonString = string_utils.PrettyJSON(jsonString)
- if !isValidJsonString {
- return jsonString
- }
-
- if passwordRegex.MatchString(jsonString) {
- result := passwordRegex.FindAllStringSubmatch(jsonString, -1)
- for _, match := range result {
- if len(match) > 1 {
- jsonString = strings.ReplaceAll(jsonString, match[1], "***")
- }
- }
- }
+ return string_utils.ReplaceAllRegexStringSubmatch(passwordRegex, jsonString, "***")
+}
- return jsonString
+// MaskByteArraysInJsonString takes a string and truncates all the instances of number array fields have the word
+// "byte" in the name. E.g. {"file_bytes": [123,68,103]} will become "{"file_bytes": [...]}"
+func MaskByteArraysInJsonString(jsonString string) string {
+ return string_utils.ReplaceAllRegexStringSubmatch(byteArrayRegex, jsonString, "...")
}
func SanitiseFields(fields map[string]interface{}) map[string]interface{} {
diff --git a/string_utils/string_utils.go b/string_utils/string_utils.go
index 39568207e484d3bd11862b5b6138fba71a46a86b..efa87313a72cf85fd3838ed6ae93fe7dc538646f 100644
--- a/string_utils/string_utils.go
+++ b/string_utils/string_utils.go
@@ -16,7 +16,14 @@ import (
"golang.org/x/text/unicode/norm"
)
-const snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+const (
+ snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+
+ regexIndexMatchStart = 0
+ regexIndexMatchEnd = 1
+ regexIndexSubmatchStart = 2
+ regexIndexSubmatchEnd = 3
+)
var snakeCaseRegex = regexp.MustCompile("^" + snakeCasePattern + "$")
@@ -49,6 +56,29 @@ func ReplaceCaseInsensitive(string, toReplace, replaceWith string) string {
return regex.ReplaceAllString(string, replaceWith)
}
+// ReplaceAllRegexStringSubmatch finds the submatches for a regular expression that has a single capturing group and
+// replaces all the submatches (i.e. the part that matches the capturing group) with replaceWith.
+// E.g. the regular expression re = a(x*)b captures any number of x's that are between an a and b.
+// ReplaceAllRegexStringSubmatch(re, "-axxb-ab-axb-x-ax-xb-ba-", "?") will result in "-a?b-a?b-a?b-x-ax-xb-ba-"
+func ReplaceAllRegexStringSubmatch(re *regexp.Regexp, s string, replaceWith string) string {
+ result := ""
+ lastIndex := 0
+
+ for _, v := range re.FindAllSubmatchIndex([]byte(s), -1) {
+ if len(v) == regexIndexSubmatchEnd+1 {
+ // One submatch - replace the submatch with replaceWith
+ result += s[lastIndex:v[regexIndexSubmatchStart]] + replaceWith + s[v[regexIndexSubmatchEnd]:v[regexIndexMatchEnd]]
+ lastIndex = v[regexIndexMatchEnd]
+ } else {
+ // A normal match with no submatch - don't replace anything (this should not really happen)
+ result += s[lastIndex:v[regexIndexMatchEnd]]
+ }
+ lastIndex = v[regexIndexMatchEnd]
+ }
+
+ return result + s[lastIndex:]
+}
+
// TrimQuotes - trims quotes from a string (ie: "foo" will return foo)
func TrimQuotes(stringToTrim string) string {
if len(stringToTrim) >= 2 {