diff --git a/README.md b/README.md index 1a5455e4972399f59416e9039ba975ae4c43e5e4..b09bb79eb78a3ae95a02e2707e800b6ac87b63c9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ make install 2. After your changes have been merged to the `main` branch of `go-utils`, run the following command which will automatically create a new tag: ``` -bob release +bob release full ``` and select project `bob-public-utils/bobgroup-go-utils` diff --git a/api_logs/api-logs.go b/api_logs/api-logs.go index 5abaefe07ec12f953bccd06812334e3ff54baa3f..33043db418d4c60e4ba4a9501ad8089e90d7710f 100644 --- a/api_logs/api-logs.go +++ b/api_logs/api-logs.go @@ -125,8 +125,11 @@ func GenerateOutgoingAPILog(startTime time.Time, requestID *string, claim map[st } typeString := "api-outgoing" - if funk.Contains(method, "webhook") { - typeString = "webhook-outgoing" + for k, v := range requestHeaders { + if strings.ToLower(k) == "x-bobgroup-type" && strings.ToLower(v) == "webhook" { + typeString = "webhook-outgoing" + break + } } apiLog := ApiLog{ diff --git a/audit/audit.go b/audit/audit.go index 39eb87fa48d3d4149916e76ac1b2eb52f3d273d5..8c8052427a1dd43b660e3ded7293b422500e0001 100644 --- a/audit/audit.go +++ b/audit/audit.go @@ -2,6 +2,7 @@ package audit import ( "encoding/json" + "fmt" "gitlab.com/uafrica/go-utils/errors" "reflect" "regexp" @@ -79,6 +80,11 @@ func GetChanges(original interface{}, new interface{}) (map[string]interface{}, index, _ := string_utils.StringToInt64(indexString) field := ToSnakeCase(change.Path[2]) + if len(change.Path) == 5 && string_utils.IsNumericString(change.Path[3]) { + // The field is actually an array of objects. + field += fmt.Sprintf("[%s] (%s)", change.Path[3], ToSnakeCase(change.Path[4])) + } + arrayObject, present := changes[objectKey] if present { if arrayOfObjects, ok := arrayObject.([]map[string]interface{}); ok { diff --git a/go.mod b/go.mod index ffb8b12c4e483a224ea3676d3a04029e456cfe0d..5a4c62654be9c4150966208b709b51206a78232d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils -go 1.18 +go 1.19 require ( github.com/MindscapeHQ/raygun4go v1.1.1 diff --git a/s3/s3.go b/s3/s3.go index 3f4cafb9c5b9ea5ab104bafbbedc84dc609730df..997f8640e568c0eda68af4c2dc9fec321b06f285 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -28,14 +28,14 @@ type S3UploadResponse struct { } type S3UploadSettings struct { - MimeType MIMEType - RetrieveSignedUrl bool - ExpiryDuration *time.Duration // Used to set expiry datetime of download links. NB: does not affect deletion of object from S3 bucket. - AddContentDisposition bool - FileName string - GenerateFileNameFromParts bool // Whether the file extension needs to be specified. If true, supply FilePrefix and FileExt. - FilePrefix string // Required when GenerateFileNameFromParts is true - FileExt string // Required when GenerateFileNameFromParts is true + MimeType MIMEType + RetrieveSignedUrl bool + ExpiryDuration *time.Duration // Used to set expiry datetime of download links. NB: does not affect deletion of object from S3 bucket. + AddContentDisposition bool + FilePath string + FileName string + FileExt string + InsertUUID bool } // Duration constants @@ -154,22 +154,39 @@ func (s SessionWithHelpers) UploadWithSettings(data []byte, bucket, fileName str // UploadWithSettingsRevised can be renamed to UploadWithSettings once original function has been deprecated. func (s SessionWithHelpers) UploadWithSettingsRevised(data []byte, bucket string, settings S3UploadSettings) (S3UploadResponse, error) { - var fileName, uploadUrl string + var fullFileName, uploadUrl string - if settings.FileName != "" { - fileName = settings.FileName + uuidString := "" + if settings.InsertUUID { + uuidString = fmt.Sprintf("_%s", uuid.New().String()) } - if settings.GenerateFileNameFromParts { - fileName = fmt.Sprintf("%s_%s.%s", settings.FilePrefix, uuid.New().String(), settings.FileExt) + + if len(settings.FileExt) > 0 { + if settings.FileExt[0] != '.' { + settings.FileExt = fmt.Sprintf(".%s", settings.FileExt) + } + } + + if len(settings.FilePath) > 0 { + if settings.FilePath[len(settings.FilePath)-1] != '/' { + settings.FilePath = fmt.Sprintf("%s/", settings.FilePath) + } + } + + fullFileName = fmt.Sprintf("%s%s%s%s", settings.FilePath, settings.FileName, uuidString, settings.FileExt) + + // Uploaded objects require a key + if fullFileName == "" { + return S3UploadResponse{}, errors.Error("no file name supplied for upload") } if settings.MimeType == "" { - settings.MimeType = getTypeForFilename(fileName) + settings.MimeType = getTypeForFilename(fullFileName) } putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), - Key: aws.String(fileName), + Key: aws.String(fullFileName), ContentType: aws.String(string(settings.MimeType)), Body: bytes.NewReader(data), } @@ -187,14 +204,10 @@ func (s SessionWithHelpers) UploadWithSettingsRevised(data []byte, bucket string if settings.RetrieveSignedUrl { var headers map[string]string - fileNameHeader := fileName - if settings.FileName != "" { - fileNameHeader = settings.FileName - } - if settings.AddContentDisposition { + headers = map[string]string{ - "content-disposition": "attachment; filename=\"" + fileNameHeader + "\"", + "content-disposition": fmt.Sprintf("attachment; filename=\"%s%s\"", settings.FileName, settings.FileExt), } } @@ -202,7 +215,8 @@ func (s SessionWithHelpers) UploadWithSettingsRevised(data []byte, bucket string if settings.ExpiryDuration != nil { downloadUrlExpiry = *settings.ExpiryDuration } - uploadUrl, err = s.GetSignedDownloadURL(bucket, fileName, downloadUrlExpiry, headers) + + uploadUrl, err = s.GetSignedDownloadURL(bucket, fullFileName, downloadUrlExpiry, headers) if err != nil { return S3UploadResponse{}, err } @@ -213,7 +227,7 @@ func (s SessionWithHelpers) UploadWithSettingsRevised(data []byte, bucket string response := S3UploadResponse{ URL: uploadUrl, - Filename: fileName, + Filename: fullFileName, Bucket: bucket, FileSize: fileSizeInBytes, } diff --git a/search/document_store.go b/search/document_store.go index 581670324dfd32bfe6faee4547aa49eb9e754795..3e4c440d4c74116fcd106391939580f5ba80277d 100644 --- a/search/document_store.go +++ b/search/document_store.go @@ -26,8 +26,11 @@ type DocumentStore struct { } // NewDocumentStore purpose: +// // create a document store index to write e.g. orders then allow one to search them +// // parameters: +// // name must be the complete openSearch index name e.g. "uafrica-v3-orders" // tmpl must be your document data struct consisting of public fields as: // Xxx string `json:"<name>" search:"keyword|text|long|date"` (can later add more types) @@ -141,6 +144,7 @@ func (ds *DocumentStore) Write(id string, data interface{}) error { // Search // Return: +// // docs will be a slice of the DocumentStore data type func (ds *DocumentStore) Search(query Query, limit int64) (res *SearchResponseHits, err error) { if ds == nil { @@ -242,15 +246,23 @@ func (ds *DocumentStore) Delete(id string) (err error) { if ds == nil { return errors.Errorf("document store == nil") } + + var delResponse *opensearchapi.Response + + defer func() { + if delResponse != nil && delResponse.Body != nil { + delResponse.Body.Close() + } + }() + del := opensearchapi.DeleteRequest{ Index: ds.name, DocumentType: "_doc", DocumentID: id, } - delResponse, err := del.Do(context.Background(), ds.w.client) + delResponse, err = del.Do(context.Background(), ds.w.client) if err != nil { - err = errors.Wrapf(err, "failed to del document") - return + return errors.Wrapf(err, "failed to del document") } switch delResponse.StatusCode { @@ -259,8 +271,8 @@ func (ds *DocumentStore) Delete(id string) (err error) { case http.StatusNoContent: default: resBody, _ := ioutil.ReadAll(delResponse.Body) - err = errors.Errorf("Del failed with HTTP status %v: %s", delResponse.StatusCode, string(resBody)) - return + return errors.Errorf("Del failed with HTTP status %v: %s", delResponse.StatusCode, string(resBody)) } + return nil } diff --git a/search/opensearch_types.go b/search/opensearch_types.go index 896541be6658c549b0ce4e57b22dd67f1a0cb6e0..8e52f073ab265d379c9978dcf78c460cc85c5cd0 100644 --- a/search/opensearch_types.go +++ b/search/opensearch_types.go @@ -39,28 +39,42 @@ type SearchRequestBody struct { Timeout string `json:"timeout,omitempty"` // timeout for search } -// Query NOTE: We are only using bool filter queries, to be able to always do the correct sorting, not based on the relevancy score // https://opensearch.org/docs/latest/opensearch/query-dsl/bool/ type Query struct { - Bool *QueryBool `json:"bool,omitempty"` + Match *QueryMatch `json:"match,omitempty" doc:"<field>:<value>"` + Term *QueryTerm `json:"term,omitempty"` + Range *QueryRange `json:"range,omitempty"` + MultiMatch *QueryMultiMatch `json:"multi_match,omitempty"` + Bool *QueryBool `json:"bool,omitempty"` + QueryString *QueryString `json:"query_string,omitempty"` + SimpleQueryString *QueryString `json:"simple_query_string,omitempty"` } type QueryBool struct { - Must []FilterQuery `json:"must,omitempty"` // List of things that must appear in matching documents and will contribute to the score. - Filter []FilterQuery `json:"filter,omitempty"` // List of things that must appear in matching documents. However, unlike must the score of the query will be ignored. Filter clauses are executed in filter context, meaning that scoring is ignored and clauses are considered for caching - Should []Query `json:"should,omitempty"` // List of things that should appear in the matching document. - MustNot []FilterQuery `json:"must_not,omitempty"` // List of things that must not appear in the matching documents. Clauses are executed in filter context meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of 0 for all documents is returned + Must []FilterQuery `json:"must,omitempty"` // List of things that must appear in matching documents and will contribute to the score. + Filter []FilterQuery `json:"filter,omitempty"` // List of things that must appear in matching documents. However, unlike must the score of the query will be ignored. Filter clauses are executed in filter context, meaning that scoring is ignored and clauses are considered for caching + Should []Query `json:"should,omitempty"` // List of things that should appear in the matching document. + MustNot []FilterQuery `json:"must_not,omitempty"` // List of things that must not appear in the matching documents. Clauses are executed in filter context meaning that scoring is ignored and clauses are considered for caching. Because scoring is ignored, a score of 0 for all documents is returned + MinimumShouldMatch int64 `json:"minimum_should_match"` } type FilterQuery struct { // one of: - Match *QueryMatch `json:"match,omitempty"` - Term *QueryTerm `json:"term,omitempty"` - Range *QueryRange `json:"range,omitempty"` - MultiMatch *QueryMultiMatch `json:"multi_match,omitempty"` - Bool *QueryBool `json:"bool,omitempty"` - QueryString *QueryString `json:"query_string,omitempty"` - Wildcard *QueryWildcard `json:"wildcard,omitempty"` // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html + Match *QueryMatch `json:"match,omitempty"` + Term *QueryTerm `json:"term,omitempty"` + Range *QueryRange `json:"range,omitempty"` + MultiMatch *QueryMultiMatch `json:"multi_match,omitempty"` + Bool *QueryBool `json:"bool,omitempty"` + QueryString *QueryString `json:"query_string,omitempty"` + SimpleQueryString *QueryString `json:"simple_query_string,omitempty"` + Wildcard *QueryWildcard `json:"wildcard,omitempty"` // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html +} + +type QueryValue struct { + Query string `json:"query"` + Operator string `json:"operator,omitempty"` // defaults to "or", accepted values: or|and + Fuzziness string `json:"fuzziness,omitempty"` // https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#fuzziness + ZeroTermsQuery string `json:"zero_terms_query,omitempty"` } type QueryMatch map[string]string @@ -70,20 +84,23 @@ type QueryTerm map[string]string type QueryWildcard map[string]string type QueryMultiMatch struct { - Query string `json:"query"` // Full value match search in selected fields - Fields []string `json:"fields,omitempty" doc:"List of fields"` - Type QueryMultiMatchType `json:"type,omitempty"` + Query string `json:"query"` // Full value match search in selected fields + Fields []string `json:"fields,omitempty" doc:"List of fields"` + Type QueryMultiMatchType `json:"type,omitempty"` + Operator string `json:"operator,omitempty"` } type QueryMultiMatchType string const ( - QueryMultiMatchTypePhrase QueryMultiMatchType = "phrase" + QueryMultiMatchTypePhrase QueryMultiMatchType = "phrase" + QueryMultiMatchTypeCrossFields QueryMultiMatchType = "cross_fields" ) type QueryString struct { - Query string `json:"query"` // Text search with partial matches, using asterisk for optional or question mark for required wildcards before and/or after text - Fields []string `json:"fields,omitempty" doc:"List of fields"` + Query string `json:"query"` // Text search with partial matches, using asterisk for optional or question mark for required wildcards before and/or after text + Fields []string `json:"fields,omitempty" doc:"List of fields"` + DefaultOperator string `json:"default_operator,omitempty"` } type QueryRange map[string]QueryExpr diff --git a/secrets_manager/secrets_manager.go b/secrets_manager/secrets_manager.go index 871e82eedb18847457a154368ec344d576c6bd88..812b5534add89f1fb04860013c0b7b6b6a7ec209 100644 --- a/secrets_manager/secrets_manager.go +++ b/secrets_manager/secrets_manager.go @@ -2,6 +2,7 @@ package secrets_manager import ( "encoding/base64" + "encoding/json" credentials2 "github.com/aws/aws-sdk-go/aws/credentials" "os" @@ -26,8 +27,8 @@ type DatabaseCredentials struct { } type S3UploadCredentials struct { - AccessKeyID string `json:"accessKeyID"` - SecretAccessKey string `json:"secretAccessKey"` + AccessKeyID string `json:"accessKeyID"` + SecretKey string `json:"secretKey"` } var ( @@ -35,6 +36,8 @@ var ( secretManagerRegion = "af-south-1" ) +var secretManagerSession *secretsmanager.SecretsManager + func GetDatabaseCredentials(secretID string, isDebug bool) (DatabaseCredentials, error) { secret, _ := GetSecret(secretID, isDebug) var credentials DatabaseCredentials @@ -52,23 +55,28 @@ func GetS3UploadCredentials(secretID string, isDebug bool) (*credentials2.Creden if err != nil { return nil, err } - return credentials2.NewStaticCredentials(credentials.AccessKeyID, credentials.SecretAccessKey, ""), nil + return credentials2.NewStaticCredentials(credentials.AccessKeyID, credentials.SecretKey, ""), nil } -func GetSecret(secretID string, isDebug bool) (string, string) { - cachedSecret, err := secretCache.GetSecretString(secretID) - if err != nil { - logs.Info("Failed to get secret key from cache") - } - if cachedSecret != "" { - return cachedSecret, "" +// getSecretManagerSession Instantiates a new Secrets Manager client session +func getSecretManagerSession(isDebug bool) (err error) { + // If a session already exists, use it + if secretManagerSession != nil { + return nil } - awsSession := session.New() + logs.Info("Creating a new Secrets Manager session") + awsSession, err := session.NewSession() + if err != nil { + return err + } // Get local config if isDebug && os.Getenv("ENVIRONMENT") != "" { - logs.Info("Using access key %s", os.Getenv("AWS_ACCESS_KEY_ID")) + awsAccessKey := os.Getenv("AWS_ACCESS_KEY_ID") + if len(awsAccessKey) > 0 { + logs.Info("Using access key %s", awsAccessKey) + } awsSession, err = session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Region: aws.String("af-south-1"), @@ -76,53 +84,53 @@ func GetSecret(secretID string, isDebug bool) (string, string) { }, }) if err != nil { - return "", "" + return err } } + // Create a Secrets Manager client session + secretManagerSession = secretsmanager.New(awsSession, aws.NewConfig().WithRegion(secretManagerRegion)) + + return nil +} + +// logError Logs any errors returned by the Secrets Manager client +func logError(err error) { + if aerr, ok := err.(awserr.Error); ok { + logs.Info(aerr.Code()+" %s", aerr.Error()) + } else { + // Print the error, cast err to awserr.Error to get the Code and + // Message from an error. + logs.Info(err.Error()) + } +} + +func GetSecret(secretID string, isDebug bool) (string, string) { + // Check if we have the secret in cache + cachedSecret, err := secretCache.GetSecretString(secretID) + if err != nil { + logs.Info("Failed to get secret key from cache") + } + if cachedSecret != "" { + return cachedSecret, "" + } + // Create a Secrets Manager client - svc := secretsmanager.New(awsSession, aws.NewConfig().WithRegion(secretManagerRegion)) + err = getSecretManagerSession(isDebug) + if err != nil { + logs.Info("Could not create client: %+v", err) + return "", "" + } + // Create a secret input := &secretsmanager.GetSecretValueInput{ SecretId: aws.String(string(secretID)), VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified } - // In this sample we only handle the specific exceptions for the 'GetSecretValue' API. - // See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html - - result, err := svc.GetSecretValue(input) + result, err := secretManagerSession.GetSecretValue(input) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case secretsmanager.ErrCodeDecryptionFailure: - // Secrets Manager can't decrypt the protected secret text using the provided KMS key. - logs.Info(secretsmanager.ErrCodeDecryptionFailure, aerr.Error()) - - case secretsmanager.ErrCodeInternalServiceError: - // An error occurred on the server side. - logs.Info(secretsmanager.ErrCodeInternalServiceError, aerr.Error()) - - case secretsmanager.ErrCodeInvalidParameterException: - // You provided an invalid value for a parameter. - logs.Info(secretsmanager.ErrCodeInvalidParameterException, aerr.Error()) - - case secretsmanager.ErrCodeInvalidRequestException: - // You provided a parameter value that is not valid for the current state of the resource. - logs.Info(secretsmanager.ErrCodeInvalidRequestException, aerr.Error()) - - case secretsmanager.ErrCodeResourceNotFoundException: - // We can't find the resource that you asked for. - logs.Info("Can't find secret with ID: ", secretID) - logs.Info(secretsmanager.ErrCodeResourceNotFoundException, aerr.Error()) - default: - logs.Info(err.Error()) - } - } else { - // Print the error, cast err to awserr.Error to get the Code and - // Message from an error. - logs.Info(err.Error()) - } + logError(err) return "", "" } @@ -135,7 +143,7 @@ func GetSecret(secretID string, isDebug bool) (string, string) { decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(result.SecretBinary))) length, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, result.SecretBinary) if err != nil { - logs.Info("Base64 Decode Error:", err) + logs.Info("Base64 Decode Error: %+v", err) return "", "" } decodedBinarySecret = string(decodedBinarySecretBytes[:length]) @@ -143,3 +151,55 @@ func GetSecret(secretID string, isDebug bool) (string, string) { return secretString, decodedBinarySecret } + +// CreateSecret Creates a JSON marshaled "string secret" (can be expanded to cater for binary secrets should the need arise) +func CreateSecret(secretID string, secret any, isDebug bool) (awsSecretID string, err error) { + // Create a Secrets Manager client + err = getSecretManagerSession(isDebug) + if err != nil { + logs.Info("Could not create client: %+v", err) + return "", err + } + + // Create the secret - marshaling "any" into a JSON string + secretStr, err := json.Marshal(secret) + if err != nil { + logs.Info("Could not marshal secret: %+v", err) + return "", err + } + input := &secretsmanager.CreateSecretInput{ + Name: aws.String(secretID), + SecretString: aws.String(string(secretStr)), + } + + result, err := secretManagerSession.CreateSecret(input) + if err != nil { + logError(err) + return "", err + } + + return aws.StringValue(result.Name), nil +} + +func DeleteSecret(secretID string, forceWithoutRecovery bool, isDebug bool) error { + // Create a Secrets Manager client + err := getSecretManagerSession(isDebug) + if err != nil { + logs.Info("Could not create client: %+v", err) + return err + } + + // Delete the secret + input := &secretsmanager.DeleteSecretInput{ + SecretId: aws.String(secretID), + ForceDeleteWithoutRecovery: aws.Bool(forceWithoutRecovery), + } + + _, err = secretManagerSession.DeleteSecret(input) + if err != nil { + logError(err) + return err + } + + return nil +} diff --git a/secrets_manager/secrets_manager_test.go b/secrets_manager/secrets_manager_test.go new file mode 100644 index 0000000000000000000000000000000000000000..3e5d15d376b119b7824996a4eab7c2e538f09986 --- /dev/null +++ b/secrets_manager/secrets_manager_test.go @@ -0,0 +1,71 @@ +package secrets_manager + +import ( + "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/date_utils" + "os" + "testing" + "time" +) + +var isDebug bool +var secretID = "TestSecret_" + time.Now().Format(date_utils.DateLayoutTrimmed()) + +func TestMain(m *testing.M) { + isDebug = true + os.Setenv("ENVIRONMENT", "dev") + os.Setenv("AWS_PROFILE", "") // <-- Use your AWS profile name here + + code := m.Run() + os.Exit(code) +} + +func TestAll(t *testing.T) { + testCreateSecret(t) + testGetSecret(t) + testDeleteSecret(t) +} + +func testCreateSecret(t *testing.T) { + type SubStruct struct { + Arg3a string + Arg3b string + } + type Anything struct { + Arg1 string + Arg2 string + Arg3 SubStruct + } + secret := Anything{ + Arg1: "lorem", + Arg2: "ipsum", + Arg3: SubStruct{ + Arg3a: "dolor", + Arg3b: "sit", + }, + } + + secretName, err := CreateSecret(secretID, secret, isDebug) + if err != nil { + t.Errorf("Secret with ID '%s' could not be created.", secretName) + } + + t.Logf("Secret with ID '%s' successfully created.", secretName) +} + +func testGetSecret(t *testing.T) { + secret, _ := GetSecret(secretID, isDebug) + if len(secret) <= 0 { + t.Errorf("Could not get secret with ID %s, or secret has no content", secretID) + } + + t.Logf("Secret with ID `%s` has content: %s", secretID, secret) +} + +func testDeleteSecret(t *testing.T) { + err := DeleteSecret(secretID, true, isDebug) + if err != nil { + t.Errorf("Secret with ID '%s' could not be deleted.", secretID) + return + } + t.Logf("Secret with ID '%s' successfully deleted.", secretID) +}