From 841babfd5497d0e287e9b020573cc83d5d249b8d Mon Sep 17 00:00:00 2001 From: jano3 <jano@bob.co.za> Date: Thu, 30 Mar 2023 11:15:01 +0200 Subject: [PATCH] #36 Sanitise logs to mask passwords --- logs/logs.go | 79 ++++++++++++++++++++++++++++++++---- string_utils/string_utils.go | 14 +++++++ 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/logs/logs.go b/logs/logs.go index ff11fcc..f1b4c59 100644 --- a/logs/logs.go +++ b/logs/logs.go @@ -3,10 +3,13 @@ package logs import ( "errors" "fmt" + "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils" "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/utils" "net/http" "net/url" "os" + "reflect" + "regexp" "runtime" "strings" @@ -27,6 +30,59 @@ var raygunClient *raygun4go.Client // TODO // Sensitive word filtering +// Password filtering +var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?",).*`) + +func SanitiseLogs(logString string) string { + logString = MaskPasswordsInJsonString(logString) + + return logString +} + +// MaskPasswordsInJsonString takes a string and, if it is a JSON string, sanitises all the password. In order for the +// regex to work correctly we need to prettify the JSON, so the function always returns a formatted JSON string. +func MaskPasswordsInJsonString(jsonString string) string { + var isValidJsonString bool + isValidJsonString, jsonString = string_utils.PrettyJSON(jsonString) + if !isValidJsonString { + return jsonString + } + + if passwordRegex.MatchString(jsonString) { + result := passwordRegex.FindAllStringSubmatch(jsonString, -1) + for _, match := range result { + if len(match) > 1 { + jsonString = strings.ReplaceAll(jsonString, match[1], "***") + } + } + } + + return jsonString +} + +func SanitiseFields(fields map[string]interface{}) map[string]interface{} { + sanitisedFields := make(map[string]interface{}) + + // Check if each field is a string or string pointer, and sanitize them if they are + for key, field := range fields { + value := reflect.ValueOf(field) + if value.Kind() == reflect.Ptr && value.IsValid() { + pointerValue := value.Elem() + if pointerValue.Kind() == reflect.String { + sanitisedString := SanitiseLogs(pointerValue.String()) + sanitisedFields[key] = &sanitisedString + } + } else if value.Kind() == reflect.String { + sanitisedFields[key] = SanitiseLogs(value.String()) + } else { + // Don't sanitise fields that + sanitisedFields[key] = field + } + } + + return sanitisedFields +} + func InitLogs(requestID *string, isDebugBuild bool, buildVersion string, request *events.APIGatewayProxyRequest, client *raygun4go.Client) { currentRequestID = requestID apiRequest = request @@ -115,16 +171,22 @@ func getLogger() *log.Entry { } func InfoWithFields(fields map[string]interface{}, message interface{}) { - getLogger().WithFields(fields).Info(message) + if reflect.TypeOf(message).Kind() == reflect.String { + message = SanitiseLogs(message.(string)) + } + sanitisedFields := SanitiseFields(fields) + getLogger().WithFields(sanitisedFields).Info(message) } func Info(format string, a ...interface{}) { - getLogger().Info(fmt.Sprintf(format, a...)) + message := SanitiseLogs(fmt.Sprintf(format, a...)) + getLogger().Info(message) } func ErrorWithFields(fields map[string]interface{}, err error) { - sendRaygunError(fields, err) - getLogger().WithFields(fields).Error(err) + sanitisedFields := SanitiseFields(fields) + sendRaygunError(sanitisedFields, err) + getLogger().WithFields(sanitisedFields).Error(err) } func ErrorWithMsg(message string, err error) { @@ -141,11 +203,13 @@ func ErrorMsg(message string) { } func Warn(format string, a ...interface{}) { - getLogger().Warn(fmt.Sprintf(format, a...)) + message := SanitiseLogs(fmt.Sprintf(format, a...)) + getLogger().Warn(message) } func WarnWithFields(fields map[string]interface{}, err error) { - getLogger().WithFields(fields).Warn(err) + sanitisedFields := SanitiseFields(fields) + getLogger().WithFields(sanitisedFields).Warn(err) } func SQLDebugInfo(sql string) { @@ -214,7 +278,8 @@ func sendRaygunError(fields map[string]interface{}, errToSend error) { } fields["env"] = env - raygunClient.CustomData(fields) + sanitisedFields := SanitiseFields(fields) + raygunClient.CustomData(sanitisedFields) raygunClient.Request(fakeHttpRequest()) if errToSend == nil { diff --git a/string_utils/string_utils.go b/string_utils/string_utils.go index 6f54c76..b649cc2 100644 --- a/string_utils/string_utils.go +++ b/string_utils/string_utils.go @@ -1,6 +1,7 @@ package string_utils import ( + "bytes" "encoding/json" "fmt" "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors" @@ -348,3 +349,16 @@ func PascalCaseToSentence(pascal string) string { return sentence } + +func PrettyJSON(jsonString string) (validJson bool, prettyString string) { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, []byte(jsonString), "", " ") + if err != nil { + validJson = false + prettyString = jsonString + } else { + validJson = true + prettyString = prettyJSON.String() + } + return +} -- GitLab