diff --git a/audit/audit.go b/audit/audit.go index 6146f093cf995d81a78bb134a07b1ee7c03f07d9..1ff39abc6c3af2287b77db1dd50bfb58970b2442 100644 --- a/audit/audit.go +++ b/audit/audit.go @@ -1,8 +1,10 @@ package audit import ( + "encoding/json" "reflect" "regexp" + "strconv" "strings" "github.com/r3labs/diff/v2" @@ -37,29 +39,9 @@ func GetChanges(original interface{}, new interface{}) (map[string]interface{}, // 0 = Object // 1 = field - objectKey := ToSnakeCase(change.Path[0]) - field := ToSnakeCase(change.Path[1]) + ChildObjectChanges(changes, change.Path[0], change.Path[1], change.From, change.To) - existingObject, present := changes[objectKey] - if present { - if object, ok := existingObject.(map[string]interface{}); ok { - object[field] = FieldChange{ - From: change.From, - To: change.To, - } - changes[objectKey] = object - } - } else { - fieldChange := map[string]interface{}{ - field: FieldChange{ - From: change.From, - To: change.To, - }, - } - changes[objectKey] = fieldChange - } - - } else if len(change.Path) == 3 { + } else if len(change.Path) >= 3 { // Array of objects // ["Parcel", "0", "ActualWeight"] // 0 = Object @@ -68,6 +50,13 @@ func GetChanges(original interface{}, new interface{}) (map[string]interface{}, objectKey := ToSnakeCase(change.Path[0]) indexString := change.Path[1] + + if !string_utils.IsNumericString(indexString) { + // Not an array, but a deeper nested object + ChildObjectChanges(changes, change.Path[len(change.Path)-2], change.Path[len(change.Path)-1], change.From, change.To) + continue + } + index, _ := string_utils.StringToInt64(indexString) field := ToSnakeCase(change.Path[2]) @@ -110,6 +99,187 @@ func GetChanges(original interface{}, new interface{}) (map[string]interface{}, return changes, nil } +func ChildObjectChanges(changes map[string]interface{}, objectPath string, fieldPath string, changeFrom interface{}, changeTo interface{}) { + + objectKey := ToSnakeCase(objectPath) + field := ToSnakeCase(fieldPath) + + existingObject, present := changes[objectKey] + if present { + if object, ok := existingObject.(map[string]interface{}); ok { + object[field] = FieldChange{ + From: changeFrom, + To: changeTo, + } + changes[objectKey] = object + } + } else { + fieldChange := map[string]interface{}{ + field: FieldChange{ + From: changeFrom, + To: changeTo, + }, + } + changes[objectKey] = fieldChange + } +} + +// GetAllChanges Returns the diff, structured in json, recursively +// Be warned, here be dragons. Debug this first to understand how it works +func GetAllChanges(original interface{}, new interface{}) (map[string]interface{}, error) { + changes := map[string]interface{}{} + changelog, err := diff.Diff(original, new) + if err != nil { + return changes, err + } + + changesJson := "{" + subArrays := map[string]string{} + for _, change := range changelog { + var value string // Keep track of the core value + var key string // Keep track of the key/s for this value + var closing string // Keep track of the correct number of closing characters + + for pathIndex := len(change.Path) - 1; pathIndex >= 0; pathIndex-- { + // If this is the first value, it's the "deepest" value of this change set, otherwise leave this var alone + // If this is set at the wrong point, other behaviour changes + if pathIndex+1 == len(change.Path) { + changedField := FieldChange{ + From: change.From, + To: change.To, + } + request, err := json.Marshal(changedField) + if err != nil { + return nil, err + } + value = string(request) + value = value[1 : len(value)-1] + } + + // If this "key" is integer-like, we handle it a little differently to ensure related data ends up + // together and formatted as you would expect JSON arrays to look + if _, convErr := strconv.Atoi(change.Path[pathIndex]); convErr == nil { + positionOfSubArray := "" + for sub := 0; sub <= pathIndex; sub++ { + positionOfSubArray = positionOfSubArray + change.Path[sub] + } + + // Add it to the placeholder data map used later on + subArrays[positionOfSubArray] = subArrays[positionOfSubArray] + key + value + closing + "," + + // Don't insert the same placeholder into the json multiple times + if !strings.Contains(changesJson, positionOfSubArray) { + value = positionOfSubArray + } else { + // Make sure this value doesn't end up populated explicitly + value = "" + } + } else { + // Safety net so we don't compare to a non existant value + if pathIndex < len(change.Path)-1 { + // If the value this one "contains" has a integer-like key, this is probably an array + if _, err = strconv.Atoi(change.Path[pathIndex+1]); err == nil { + key = "\"" + ToSnakeCase(change.Path[pathIndex]) + "\": [" + closing = "]" + } else { + key = "\"" + ToSnakeCase(change.Path[pathIndex]) + "\": {" + key + closing = closing + "}" + } + } else { + key = "\"" + ToSnakeCase(change.Path[pathIndex]) + "\": {" + key + closing = closing + "}" + } + } + } + + // Don't insert empty values (happens when multiple values within an array are set, see "placeholder" behavior) + if value != "" { + // Duplicate key prevention + keySplit := strings.Split(key, ": ") + lastOccurrence := 0 + lastMatched := "" + levelsDeep := 0 + for sIndex, split := range keySplit { + // Trim leading character off split + if split[0:1] == "{" || split[0:1] == "[" { + split = split[1:] + } + + // The final value of keySplit might be empty ("") + if split != "" { + if index := strings.Index(changesJson, split); index > -1 && changesJson != "{" { + // Prevent reverse traversal + if index > lastOccurrence { + // Prevent finding nested keys as opposed to our "change" keys + opener := changesJson[index+len(split)+2 : index+len(split)+3] + // Prevent a "base level" key from matching a nested key + baseLevel := false + if index > 1 { + containedBy := changesJson[index-1 : index] + if levelsDeep == 0 && (containedBy == "{" || containedBy == "[") { + baseLevel = true + } + } + if (opener == "{" || opener == "[") && !baseLevel { + lastOccurrence = index + lastMatched = split + levelsDeep = sIndex + } + } + } + } + } + + // If the "key" is already present, handle it differently + if lastOccurrence > 0 { + // Strip parent keys that are already present + key = key[strings.Index(key, lastMatched)+len(lastMatched):] + if key[0:2] == ": " { + key = key[2:] + } + // Strip the correct amount of closing tags + closing = closing[:len(closing)-levelsDeep] + + // Find the position of this key in the master string + position := strings.Index(changesJson, lastMatched) + len(lastMatched) + 3 + + // We're appending to an existing object, so strip the outermost wrapping layer + key = key[1:] + closing = closing[:len(closing)-1] + + // Place the value within the existing key + changesJson = changesJson[:position] + key + value + closing + ", " + changesJson[position:] + } else { + // No value found, append it to the end + changesJson = changesJson + key + value + closing + ", " + } + } + } + // Trim whitespace and strip trailing comma since we are done inserting at the back + changesJson = strings.TrimSpace(changesJson) + if changesJson[len(changesJson)-1:] == "," { + changesJson = changesJson[:len(changesJson)-1] + } + changesJson = changesJson + "}" + + // Now we can go make sure the placeholders are populated with the data in + for placeholderKey, placeholderValue := range subArrays { + // Trim the trailing comma since we won't be adding any more stuff + placeholderValue = strings.TrimSpace(placeholderValue) + if placeholderValue[len(placeholderValue)-1:] == "," { + placeholderValue = placeholderValue[:len(placeholderValue)-1] + } + changesJson = strings.ReplaceAll(changesJson, placeholderKey, "{"+placeholderValue+"}") + } + + // Now, hopefully, the json parsing will pass and we have a nicely formatted dataset + err = json.Unmarshal([]byte(changesJson), &changes) + if err != nil { + return nil, err + } + + return changes, nil +} var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") diff --git a/audit/audit_test.go b/audit/audit_test.go new file mode 100644 index 0000000000000000000000000000000000000000..b10fce6717ebfbc9ba78ed55f08eb5f981b81a98 --- /dev/null +++ b/audit/audit_test.go @@ -0,0 +1,46 @@ +package audit + +import ( + "encoding/json" + "fmt" + "testing" +) + +type MockObject struct { + Object interface{} +} + +func TestGetChanges(t *testing.T) { + + object1 := MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: "done", + }, + }, + }, + }, + } + + object2 := MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: MockObject{ + Object: "done1", + }, + }, + }, + }, + } + + changes, err := GetChanges(object1, object2) + if err != nil { + panic(err) + } + + result, _ := json.Marshal(changes) + fmt.Println(result) +} diff --git a/s3/s3.go b/s3/s3.go index 478d1badd7339b9012f46fd6f00962330685a013..35d7a7fd7c93ecdb5c1681332662a5ef9d770695 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -32,6 +32,7 @@ type S3UploadSettings struct { RetrieveSignedUrl bool ExpiryDuration *time.Duration AddContentDisposition bool + FileName string } type MIMEType string @@ -123,9 +124,15 @@ func (s SessionWithHelpers) UploadWithSettings(data []byte, bucket, fileName str if settings.RetrieveSignedUrl { var headers map[string]string + + fileNameHeader := fileName + if settings.FileName != "" { + fileNameHeader = settings.FileName + } + if settings.AddContentDisposition { headers = map[string]string{ - "content-disposition": "attachment; filename=\"" + fileName + "\"", + "content-disposition": "attachment; filename=\"" + fileNameHeader + "\"", } } @@ -135,16 +142,17 @@ func (s SessionWithHelpers) UploadWithSettings(data []byte, bucket, fileName str return "", nil } -func (s SessionWithHelpers) UploadWith1DayExpiry(data []byte, bucket, fileName string, mimeType MIMEType) (string, error) { +func (s SessionWithHelpers) UploadWith1DayExpiry(data []byte, bucket, fileName string, mimeType MIMEType, shouldDownloadInsteadOfOpen bool) (string, error) { if mimeType == "" { mimeType = getTypeForFilename(fileName) } expiry := 24 * time.Hour signedUrl, err := s.UploadWithSettings(data, bucket, fileName, S3UploadSettings{ - MimeType: mimeType, - RetrieveSignedUrl: true, - ExpiryDuration: &expiry, + MimeType: mimeType, + RetrieveSignedUrl: true, + ExpiryDuration: &expiry, + AddContentDisposition: shouldDownloadInsteadOfOpen, }) if err != nil { return "", err diff --git a/secrets_manager/secrets_manager.go b/secrets_manager/secrets_manager.go index 171ce187915fc8c1cf12a3d433a133495041f7aa..4f038c207479fb7e5fe89541a121a2c16c6f04ba 100644 --- a/secrets_manager/secrets_manager.go +++ b/secrets_manager/secrets_manager.go @@ -21,6 +21,7 @@ type DatabaseCredentials struct { Host string `json:"host"` Port int `json:"port"` InstanceIdentifier string `json:"dbInstanceIdentifier"` + ReadOnlyHost string `json:"aurora_read_only_host"` } var ( diff --git a/slice_utils/slice_utils.go b/slice_utils/slice_utils.go index ce2b362d8aade32f81675ff63da27caa000b487d..0c400cb7863bf42ead48dfee4ee6b57464347de5 100644 --- a/slice_utils/slice_utils.go +++ b/slice_utils/slice_utils.go @@ -1,7 +1,7 @@ package slice_utils import ( -"github.com/thoas/go-funk" + "github.com/thoas/go-funk" ) func MinimumFloat64(values []float64) (min float64) { @@ -37,3 +37,19 @@ func ElementExists(in interface{}, elem interface{}) (bool, int) { idx := funk.IndexOf(in, elem) return idx != -1, idx } + +func FilterNonZero(arr []int64) []int64 { + // Filter out the zero numbers + nonZeroNumbers := funk.Filter(arr, func(number int64) bool { + return number != 0 + }).([]int64) + return nonZeroNumbers +} + +func FilterNonEmptyString(arr []string) []string { + // Filter out empty strings + nonEmptyStrings := funk.Filter(arr, func(value string) bool { + return value != "" + }).([]string) + return nonEmptyStrings +} diff --git a/sqs/sqs.go b/sqs/sqs.go index e9fa2f6f3554f5011923d64c01d0912c4770c057..be6deb7cc036ff7ffa2f43aedf6d150d35ca476d 100644 --- a/sqs/sqs.go +++ b/sqs/sqs.go @@ -7,7 +7,6 @@ import ( "fmt" "github.com/google/uuid" "gitlab.com/uafrica/go-utils/s3" - "gitlab.com/uafrica/go-utils/string_utils" "io/ioutil" "time" @@ -29,6 +28,7 @@ type Messenger struct { S3Session *s3.SessionWithHelpers S3BucketName string MessageGroupID *string + DelaySeconds *int64 RequestIDHeaderKey string } @@ -79,22 +79,21 @@ func (m *Messenger) SendSQSMessage(headers map[string]string, body string, curre StringValue: aws.String(sqsType), } + // SQS has max of 15 minutes delay + // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html + if m.DelaySeconds != nil && aws.Int64Value(m.DelaySeconds) > 900 { + m.DelaySeconds = aws.Int64(900) + } + var res *sqs.SendMessageOutput var err error - if string_utils.UnwrapString(m.MessageGroupID) == "" { - res, err = sqsClient.SendMessage(&sqs.SendMessageInput{ - MessageAttributes: msgAttrs, - MessageBody: aws.String(body), - QueueUrl: &m.QueueURL, - }) - } else { - res, err = sqsClient.SendMessage(&sqs.SendMessageInput{ - MessageAttributes: msgAttrs, - MessageBody: aws.String(body), - QueueUrl: &m.QueueURL, - MessageGroupId: m.MessageGroupID, - }) - } + res, err = sqsClient.SendMessage(&sqs.SendMessageInput{ + MessageAttributes: msgAttrs, + MessageBody: aws.String(body), + QueueUrl: &m.QueueURL, + MessageGroupId: m.MessageGroupID, + DelaySeconds: m.DelaySeconds, + }) if err != nil { return "", err diff --git a/string_utils/string_utils.go b/string_utils/string_utils.go index 55677a7d21c545540049a2c2857af0dcb89fb582..14d7bcf7567cefbfc452290e0940ec910125667c 100644 --- a/string_utils/string_utils.go +++ b/string_utils/string_utils.go @@ -294,3 +294,20 @@ func LimitStringToMaxLength(str string, maxLen int) string { } return str } + +// StripQueryString - Strips the query parameters from a URL +func StripQueryString(inputUrl string) (string, error) { + u, err := url.Parse(inputUrl) + if err != nil { + return inputUrl, err + } + u.RawQuery = "" + return u.String(), nil +} + +func IsValidEmail(email string) bool { + // Found here: https://stackoverflow.com/questions/201323/how-can-i-validate-an-email-address-using-a-regular-expression + regex, _ := regexp.Compile("(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])") + isValidEmail := regex.MatchString(email) + return isValidEmail +} diff --git a/string_utils/string_utils_test.go b/string_utils/string_utils_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3c52e65fd4ae909cc7a7dda123d7d8d3c4adbca6 --- /dev/null +++ b/string_utils/string_utils_test.go @@ -0,0 +1,37 @@ +package string_utils + +import "testing" + +func TestIsValidEmail(t *testing.T) { + type args struct { + email string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "valid", + args: args{email: "johandk@uafrica.com"}, + want: true, + }, + { + name: "invalid", + args: args{email: "johandk@@uafrica.com"}, + want: false, + }, + { + name: "invalid", + args: args{email: "johandk@uafricacom"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsValidEmail(tt.args.email); got != tt.want { + t.Errorf("IsValidEmail() = %v, want %v", got, tt.want) + } + }) + } +}