Merge branch 'main' of gitlab.bob.co.za:bob-public-utils/bobgroup-go-utils

go.mod

 module gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils
 
-go 1.19
+go 1.21
 
 require (
 	github.com/MindscapeHQ/raygun4go v1.1.1

handler_utils/request.go

@@ -15,7 +15,7 @@ import (
 	"github.com/aws/aws-lambda-go/lambdacontext"
 )
 
-const HTTPXRequestIDHeaderValue = "X-Request-ID"
+const HTTPXRequestIDHeaderKey = "X-Request-ID"
 
 func RequestIDFromLambdaContext(ctx context.Context) *string {
 	// Get request ID from context
@@ -39,14 +39,14 @@ func RequestIDFromHeaders(headers map[string]string, requestIDHeaderKey string)
 func AddRequestIDToHeaders(requestID *string, responseHeaders map[string]string, requestIDHeaderKey string, requestHeaders map[string]string) {
 	if requestID != nil && responseHeaders != nil {
 		responseHeaders[requestIDHeaderKey] = *requestID
-		responseHeaders[HTTPXRequestIDHeaderValue] = *requestID
+		responseHeaders[HTTPXRequestIDHeaderKey] = *requestID
 	}
 
 	// Add the HTTP X-Request-ID request header to the response headers: https://http.dev/x-request-id
 	for key, val := range requestHeaders {
 		// Don't be case-sensitive
-		if strings.ToLower(key) == strings.ToLower(HTTPXRequestIDHeaderValue) {
-			responseHeaders[key] = val
+		if strings.ToLower(key) == strings.ToLower(HTTPXRequestIDHeaderKey) {
+			responseHeaders[HTTPXRequestIDHeaderKey] = val
 			break
 		}
 	}

logs/database_logs.go

@@ -31,6 +31,7 @@ var ignoredTableInserts = []string{
 }
 
 type QueryHook struct {
+	IgnoredTableInserts []string
 	Debug               bool
 }
 
@@ -47,8 +48,9 @@ func (d QueryHook) AfterQuery(_ context.Context, event *bun.QueryEvent) {
 	shouldLogQuery := !strings.Contains(sqlQuery, "api_key")
 
 	// Don't log queries for certain tables
+	ignoredTableInsertsCombined := append(ignoredTableInserts, d.IgnoredTableInserts...)
 	tableName := TableNameForQuery(event)
-	if lo.Contains(ignoredTableInserts, tableName) {
+	if lo.Contains(ignoredTableInsertsCombined, tableName) {
 		shouldLogQuery = false
 	}
 

logs/logs.go

@@ -30,33 +30,32 @@ var build string
 var raygunClient *raygun4go.Client
 
 // Password filtering
-var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?",).*`)
+var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?").*`)
+var byteArrayRegex = regexp.MustCompile(`(?i:\\?"(?i:[\w]*)(?i:byte|data)(?i:[\w]*)\\?"\s*:\s*\[([\d\s,]+)*\])`)
 
 func SanitiseLogs(logString string) string {
+	var isValidJsonString bool
+	isValidJsonString, logString = string_utils.PrettyJSON(logString)
+	if !isValidJsonString {
+		return logString
+	}
+
+	logString = MaskByteArraysInJsonString(logString)
 	logString = MaskPasswordsInJsonString(logString)
 
 	return logString
 }
 
-// MaskPasswordsInJsonString takes a string and, if it is a JSON string, sanitises all the password. In order for the
-// regex to work correctly we need to prettify the JSON, so the function always returns a formatted JSON string.
+// MaskPasswordsInJsonString takes a string and sanitises all the instances of fields named password.
+// E.g. "{"password": "xyz123"}" will become "{"password": "***"}"
 func MaskPasswordsInJsonString(jsonString string) string {
-	var isValidJsonString bool
-	isValidJsonString, jsonString = string_utils.PrettyJSON(jsonString)
-	if !isValidJsonString {
-		return jsonString
-	}
-
-	if passwordRegex.MatchString(jsonString) {
-		result := passwordRegex.FindAllStringSubmatch(jsonString, -1)
-		for _, match := range result {
-			if len(match) > 1 {
-				jsonString = strings.ReplaceAll(jsonString, match[1], "***")
-			}
-		}
+	return string_utils.ReplaceAllRegexStringSubmatch(passwordRegex, jsonString, "***")
 }
 
-	return jsonString
+// MaskByteArraysInJsonString takes a string and truncates all the instances of number array fields have the word
+// "byte" in the name. E.g. {"file_bytes": [123,68,103]} will become "{"file_bytes": [...]}"
+func MaskByteArraysInJsonString(jsonString string) string {
+	return string_utils.ReplaceAllRegexStringSubmatch(byteArrayRegex, jsonString, "...")
 }
 
 func SanitiseFields(fields map[string]interface{}) map[string]interface{} {

string_utils/string_utils.go

@@ -16,7 +16,14 @@ import (
 	"golang.org/x/text/unicode/norm"
 )
 
-const snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+const (
+	snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+
+	regexIndexMatchStart    = 0
+	regexIndexMatchEnd      = 1
+	regexIndexSubmatchStart = 2
+	regexIndexSubmatchEnd   = 3
+)
 
 var snakeCaseRegex = regexp.MustCompile("^" + snakeCasePattern + "$")
 
@@ -49,6 +56,29 @@ func ReplaceCaseInsensitive(string, toReplace, replaceWith string) string {
 	return regex.ReplaceAllString(string, replaceWith)
 }
 
+// ReplaceAllRegexStringSubmatch finds the submatches for a regular expression that has a single capturing group and
+// replaces all the submatches (i.e. the part that matches the capturing group) with replaceWith.
+// E.g. the regular expression re = a(x*)b captures any number of x's that are between an a and b.
+// ReplaceAllRegexStringSubmatch(re, "-axxb-ab-axb-x-ax-xb-ba-", "?") will result in "-a?b-a?b-a?b-x-ax-xb-ba-"
+func ReplaceAllRegexStringSubmatch(re *regexp.Regexp, s string, replaceWith string) string {
+	result := ""
+	lastIndex := 0
+
+	for _, v := range re.FindAllSubmatchIndex([]byte(s), -1) {
+		if len(v) == regexIndexSubmatchEnd+1 {
+			// One submatch - replace the submatch with replaceWith
+			result += s[lastIndex:v[regexIndexSubmatchStart]] + replaceWith + s[v[regexIndexSubmatchEnd]:v[regexIndexMatchEnd]]
+			lastIndex = v[regexIndexMatchEnd]
+		} else {
+			// A normal match with no submatch - don't replace anything (this should not really happen)
+			result += s[lastIndex:v[regexIndexMatchEnd]]
+		}
+		lastIndex = v[regexIndexMatchEnd]
+	}
+
+	return result + s[lastIndex:]
+}
+
 // TrimQuotes - trims quotes from a string (ie: "foo" will return foo)
 func TrimQuotes(stringToTrim string) string {
 	if len(stringToTrim) >= 2 {