Hide byte arrays when sanitising json strings

logs/logs.go

@@ -30,33 +30,32 @@ var build string
 var raygunClient *raygun4go.Client
 
 // Password filtering
-var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?",).*`)
+var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?").*`)
+var byteArrayRegex = regexp.MustCompile(`(?i:\\?"(?i:[\w]*)byte(?i:[\w]*)\\?"\s*:\s*\[([\d\s,]+)*\])`)
 
 func SanitiseLogs(logString string) string {
+	var isValidJsonString bool
+	isValidJsonString, logString = string_utils.PrettyJSON(logString)
+	if !isValidJsonString {
+		return logString
+	}
+
+	logString = MaskByteArraysInJsonString(logString)
 	logString = MaskPasswordsInJsonString(logString)
 
 	return logString
 }
 
-// MaskPasswordsInJsonString takes a string and, if it is a JSON string, sanitises all the password. In order for the
-// regex to work correctly we need to prettify the JSON, so the function always returns a formatted JSON string.
+// MaskPasswordsInJsonString takes a string and sanitises all the instances of fields named password.
+// E.g. "{"password": "xyz123"}" will become "{"password": "***"}"
 func MaskPasswordsInJsonString(jsonString string) string {
-	var isValidJsonString bool
-	isValidJsonString, jsonString = string_utils.PrettyJSON(jsonString)
-	if !isValidJsonString {
-		return jsonString
-	}
-
-	if passwordRegex.MatchString(jsonString) {
-		result := passwordRegex.FindAllStringSubmatch(jsonString, -1)
-		for _, match := range result {
-			if len(match) > 1 {
-				jsonString = strings.ReplaceAll(jsonString, match[1], "***")
-			}
-		}
+	return string_utils.ReplaceAllRegexStringSubmatch(passwordRegex, jsonString, "***")
 }
 
-	return jsonString
+// MaskByteArraysInJsonString takes a string and truncates all the instances of number array fields have the word
+// "byte" in the name. E.g. {"file_bytes": [123,68,103]} will become "{"file_bytes": [...]}"
+func MaskByteArraysInJsonString(jsonString string) string {
+	return string_utils.ReplaceAllRegexStringSubmatch(byteArrayRegex, jsonString, "...")
 }
 
 func SanitiseFields(fields map[string]interface{}) map[string]interface{} {

string_utils/string_utils.go

@@ -16,7 +16,14 @@ import (
 	"golang.org/x/text/unicode/norm"
 )
 
-const snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+const (
+	snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+
+	regexIndexMatchStart    = 0
+	regexIndexMatchEnd      = 1
+	regexIndexSubmatchStart = 2
+	regexIndexSubmatchEnd   = 3
+)
 
 var snakeCaseRegex = regexp.MustCompile("^" + snakeCasePattern + "$")
 
@@ -49,6 +56,29 @@ func ReplaceCaseInsensitive(string, toReplace, replaceWith string) string {
 	return regex.ReplaceAllString(string, replaceWith)
 }
 
+// ReplaceAllRegexStringSubmatch finds the submatches for a regular expression that has a single capturing group and
+// replaces all the submatches (i.e. the part that matches the capturing group) with replaceWith.
+// E.g. the regular expression re = a(x*)b captures any number of x's that are between an a and b.
+// ReplaceAllRegexStringSubmatch(re, "-axxb-ab-axb-x-ax-xb-ba-", "?") will result in "-a?b-a?b-a?b-x-ax-xb-ba-"
+func ReplaceAllRegexStringSubmatch(re *regexp.Regexp, s string, replaceWith string) string {
+	result := ""
+	lastIndex := 0
+
+	for _, v := range re.FindAllSubmatchIndex([]byte(s), -1) {
+		if len(v) == regexIndexSubmatchEnd+1 {
+			// One submatch - replace the submatch with replaceWith
+			result += s[lastIndex:v[regexIndexSubmatchStart]] + replaceWith + s[v[regexIndexSubmatchEnd]:v[regexIndexMatchEnd]]
+			lastIndex = v[regexIndexMatchEnd]
+		} else {
+			// A normal match with no submatch - don't replace anything (this should not really happen)
+			result += s[lastIndex:v[regexIndexMatchEnd]]
+		}
+		lastIndex = v[regexIndexMatchEnd]
+	}
+
+	return result + s[lastIndex:]
+}
+
 // TrimQuotes - trims quotes from a string (ie: "foo" will return foo)
 func TrimQuotes(stringToTrim string) string {
 	if len(stringToTrim) >= 2 {