diff --git a/audit/audit.go b/audit/audit.go new file mode 100644 index 0000000000000000000000000000000000000000..6d2968832be7f6b62b5a4f83b7e984bc59f66234 --- /dev/null +++ b/audit/audit.go @@ -0,0 +1,183 @@ +package audit + +import ( + "reflect" + "regexp" + "strings" + "time" + + "github.com/r3labs/diff/v2" + "gitlab.com/uafrica/go-utils/reflection" + "gitlab.com/uafrica/go-utils/string_utils" +) + +type Event struct { + ID int64 `json:"id"` + ObjectID string `json:"object_id"` + Type string `json:"type"` + ProviderID int64 `json:"provider_id"` + Source string `json:"source"` + Timestamp time.Time `json:"timestamp"` + Change map[string]interface{} `json:"change"` +} + +type FieldChange struct { + From interface{} `json:"change_from"` + To interface{} `json:"change_to"` +} + +func GetAuditEvent(original interface{}, new interface{}, username *string) (Event, error) { + changelog, err := diff.Diff(original, new) + if err != nil { + return Event{}, err + } + + changes := map[string]interface{}{} + for _, change := range changelog { + + if len(change.Path) == 1 { + // Root object change + field := ToSnakeCase(change.Path[0]) + changes[field] = FieldChange{ + From: change.From, + To: change.To, + } + } else if len(change.Path) == 2 { + // Child object changed + // ["Account", "ID"] + // 0 = Object + // 1 = field + + objectKey := ToSnakeCase(change.Path[0]) + field := ToSnakeCase(change.Path[1]) + + existingObject, present := changes[objectKey] + if present { + if object, ok := existingObject.(map[string]interface{}); ok { + object[field] = FieldChange{ + From: change.From, + To: change.To, + } + changes[objectKey] = object + } + } else { + fieldChange := map[string]interface{}{ + field: FieldChange{ + From: change.From, + To: change.To, + }, + } + changes[objectKey] = fieldChange + } + + } else if len(change.Path) == 3 { + // Array of objects + // ["Parcel", "0", "ActualWeight"] + // 0 = Object + // 1 = Index of object + // 2 = field + + objectKey := ToSnakeCase(change.Path[0]) + indexString := change.Path[1] + index, _ := string_utils.StringToInt64(indexString) + field := ToSnakeCase(change.Path[2]) + + arrayObject, present := changes[objectKey] + if present { + if arrayOfObjects, ok := arrayObject.([]map[string]interface{}); ok { + if len(arrayOfObjects) > int(index) { + // Add field to existing object in array + object := arrayOfObjects[index] + object[field] = FieldChange{ + From: change.From, + To: change.To, + } + } else { + // new object, append to existing array + fieldChange := map[string]interface{}{ + field: FieldChange{ + From: change.From, + To: change.To, + }, + } + changes[objectKey] = append(arrayOfObjects, fieldChange) + } + + } + } else { + // Create array of objects + fieldChange := map[string]interface{}{ + field: FieldChange{ + From: change.From, + To: change.To, + }, + } + changes[objectKey] = []map[string]interface{}{ + fieldChange, + } + } + } + } + + objectID := getIntValue(original, "ID") + + if objectID == 0 { + objectID = getIntValue(new, "ID") + } + + objectIDString := string_utils.Int64ToString(objectID) + + if objectIDString == "0" { + objectIDString = getStringValue(original, "Username") + } + + if objectIDString == "" { + objectIDString = getStringValue(new, "Username") + } + + providerID := getIntValue(original, "ProviderID") + if providerID == 0 { + providerID = getIntValue(new, "ProviderID") + } + + auditEvent := Event{ + ObjectID: objectIDString, + ProviderID: providerID, + Source: "SYSTEM", + Timestamp: time.Now(), + Change: changes, + } + if username != nil { + auditEvent.Source = *username + } + return auditEvent, nil +} + +var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)") +var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])") + +func ToSnakeCase(str string) string { + snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}") + snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}") + return strings.ToLower(snake) +} + +func getIntValue(object interface{}, key string) int64 { + structValue := reflect.ValueOf(object) + if structValue.Kind() == reflect.Struct { + field := structValue.FieldByName(key) + id := reflection.GetInt64Value(field) + return id + } + return 0 +} + +func getStringValue(object interface{}, key string) string { + structValue := reflect.ValueOf(object) + if structValue.Kind() == reflect.Struct { + field := structValue.FieldByName(key) + id := reflection.GetStringValue(field) + return id + } + return "" +} diff --git a/cognito/cognito.go b/cognito/cognito.go new file mode 100644 index 0000000000000000000000000000000000000000..afae2c91b5821bf879cac0ef7fc9d1d07087be1c --- /dev/null +++ b/cognito/cognito.go @@ -0,0 +1,135 @@ +package cognito + +import ( + "fmt" + "math/rand" + "strings" + + "gitlab.com/uafrica/go-utils/logs" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-sdk-go/service/cognitoidentityprovider" +) + +var CognitoService *cognitoidentityprovider.CognitoIdentityProvider + +// ------------------------------------------------------------------------------------------------ +// Groups +// ------------------------------------------------------------------------------------------------ + +func AddCognitoUserToGroup(username string, pool string, group string) error { + groupInput := cognitoidentityprovider.AdminAddUserToGroupInput{ + GroupName: &group, + UserPoolId: &pool, + Username: &username, + } + groupOutput, err := CognitoService.AdminAddUserToGroup(&groupInput) + logs.Info("groupOutput", groupOutput) + return err +} + +func RemoveCognitoUserFromGroup(username string, pool string, group string) error { + groupInput := cognitoidentityprovider.AdminRemoveUserFromGroupInput{ + GroupName: &group, + UserPoolId: &pool, + Username: &username, + } + groupOutput, err := CognitoService.AdminRemoveUserFromGroup(&groupInput) + logs.Info("groupOutput", groupOutput) + return err +} + +// ------------------------------------------------------------------------------------------------ +// Users +// ------------------------------------------------------------------------------------------------ + +func GetCognitoUser(pool string, username string) (*cognitoidentityprovider.AdminGetUserOutput, error) { + userInput := cognitoidentityprovider.AdminGetUserInput{ + UserPoolId: &pool, + Username: &username, + } + + userOutput, err := CognitoService.AdminGetUser(&userInput) + if err != nil { + return nil, err + } + return userOutput, nil +} + +// Deletes a user from its cognito user pool +func DeleteCognitoUser(userPoolId string, username string) error { + input := cognitoidentityprovider.AdminDeleteUserInput{ + UserPoolId: &userPoolId, + Username: &username, + } + output, err := CognitoService.AdminDeleteUser(&input) + logs.Info("output", output) + return err +} + +func ResetUserPassword(pool string, username string) (*cognitoidentityprovider.AdminResetUserPasswordOutput, error) { + input := cognitoidentityprovider.AdminResetUserPasswordInput{ + UserPoolId: &pool, + Username: &username, + } + output, err := CognitoService.AdminResetUserPassword(&input) + logs.Info("output", output) + return output, err +} + +func SetUserPassword(pool string, username string, password string) (*cognitoidentityprovider.AdminSetUserPasswordOutput, error) { + setPermanently := true + input := cognitoidentityprovider.AdminSetUserPasswordInput{ + UserPoolId: &pool, + Username: &username, + Password: &password, + Permanent: &setPermanently, + } + output, err := CognitoService.AdminSetUserPassword(&input) + logs.Info("output", output) + return output, err +} + +// FOR API LOGS + +func DetermineAuthType(identity events.APIGatewayRequestIdentity) *string { + result := "cognito" + if identity.CognitoAuthenticationType == "" { + result = "iam" + } + + return &result +} + +func GetAuthUsername(identity events.APIGatewayRequestIdentity) string { + if identity.CognitoAuthenticationProvider != "" { + split := strings.Split(identity.CognitoAuthenticationProvider, ":") + return split[len(split)-1] + } + + // IAM + split := strings.Split(identity.UserArn, ":user/") + return split[len(split)-1] +} + +// Create a pseudorandom password consisting of two three-letter words and two digits +func RandomPassword() string { + i := rand.Intn(100) + var j int + for { + j = rand.Intn(100) + if j != i { + break + } + } + return fmt.Sprintf("%s%s%s", words[i], words[j], RandomDigitString(2)) +} + +// Create a pseudorandom string of digits (0-9) with specified length +func RandomDigitString(len int) string { + var str strings.Builder + for i := 0; i < len; i++ { + fmt.Fprintf(&str, "%v", rand.Intn(10)) + } + return str.String() +} diff --git a/cognito/words.go b/cognito/words.go new file mode 100644 index 0000000000000000000000000000000000000000..72c46fbe66384aac4350ec1225f7924bc8af8fbb --- /dev/null +++ b/cognito/words.go @@ -0,0 +1,104 @@ +package cognito + +var words = []string{ + "act", + "add", + "age", + "air", + "ant", + "ark", + "arm", + "art", + "ash", + "bad", + "bag", + "bar", + "bat", + "bay", + "bed", + "bee", + "big", + "box", + "bus", + "bye", + "can", + "car", + "cat", + "con", + "cow", + "cup", + "day", + "dog", + "ear", + "end", + "eye", + "far", + "fly", + "fox", + "fry", + "fun", + "gap", + "gym", + "hat", + "hip", + "hop", + "ice", + "ink", + "jam", + "jar", + "joy", + "key", + "kit", + "lab", + "law", + "log", + "low", + "man", + "max", + "may", + "net", + "nut", + "oil", + "one", + "out", + "owl", + "own", + "pen", + "pet", + "pie", + "pig", + "pin", + "pro", + "ram", + "rap", + "ray", + "red", + "row", + "sea", + "see", + "sew", + "sit", + "six", + "sky", + "son", + "spy", + "tan", + "tax", + "tea", + "ted", + "tee", + "ten", + "the", + "tin", + "toe", + "top", + "toy", + "tub", + "two", + "way", + "wig", + "yes", + "you", + "zip", + "zoo", +} diff --git a/go.mod b/go.mod index 36785627e67919074dc9a4b6cebe91133a0a01af..059784eb9fb2ea7b1a244a7e9bb3d2586104d850 100644 --- a/go.mod +++ b/go.mod @@ -5,31 +5,40 @@ go 1.17 require ( github.com/MindscapeHQ/raygun4go v1.1.1 github.com/aws/aws-lambda-go v1.26.0 + github.com/aws/aws-sdk-go v1.42.12 + github.com/aws/aws-secretsmanager-caching-go v1.1.0 github.com/go-pg/pg/v10 v10.10.6 - github.com/go-redis/redis/v8 v8.11.3 + github.com/go-redis/redis/v8 v8.11.4 + github.com/go-redis/redis_rate/v9 v9.1.2 github.com/google/uuid v1.3.0 github.com/opensearch-project/opensearch-go v1.0.0 github.com/pkg/errors v0.9.1 + github.com/r3labs/diff/v2 v2.14.2 github.com/sirupsen/logrus v1.8.1 github.com/thoas/go-funk v0.9.1 golang.org/x/text v0.3.7 ) require ( - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-errors/errors v1.4.1 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pborman/uuid v1.2.1 // indirect github.com/smartystreets/goconvey v1.7.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/vmihailenco/bufpool v0.1.11 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect + google.golang.org/appengine v1.6.6 // indirect + google.golang.org/protobuf v1.26.0 // indirect mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 363087092acd50e62a678e813e498b3341d797f5..8e1135889338c3f3e817436a06af7e35550d4014 100644 --- a/go.sum +++ b/go.sum @@ -4,9 +4,14 @@ github.com/MindscapeHQ/raygun4go v1.1.1 h1:fk3Uknv9kQxUIwL3mywwHQRyfq3PaR9lE/e40 github.com/MindscapeHQ/raygun4go v1.1.1/go.mod h1:NW0eWi2Qs00ZcctO6owrVMY+h2HxzJVgQGDrTj2ysw4= github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA= github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= +github.com/aws/aws-sdk-go v1.19.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA= +github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-secretsmanager-caching-go v1.1.0 h1:vcV94XGJ9KouXKYBTMqgrBw96Tae8JKLmoUZ5SbaXNo= +github.com/aws/aws-secretsmanager-caching-go v1.1.0/go.mod h1:wahQpJP1dZKMqjGFAjGCqilHkTlN0zReGWocPLbXmxg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -26,12 +31,15 @@ github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso= github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= -github.com/go-redis/redis/v8 v8.11.3 h1:GCjoYp8c+yQTJfc0n69iwSiHjvuAdruxl7elnZCxgt8= -github.com/go-redis/redis/v8 v8.11.3/go.mod h1:xNJ9xDG09FsIPwh3bWdk+0oDWHbtF9rPN0F/oD9XeKc= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ= +github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -42,6 +50,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -49,6 +58,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -58,6 +68,11 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -77,8 +92,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opensearch-project/opensearch-go v1.0.0 h1:8Gh7B7Un5BxuxWAgmzleEF7lpOtC71pCgPp7lKr3ca8= github.com/opensearch-project/opensearch-go v1.0.0/go.mod h1:FrUl/52DBegRYvK7ISF278AXmjDV647lyTnsLGBR7J4= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= @@ -88,6 +103,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/r3labs/diff/v2 v2.14.2 h1:1HVhQKwg1YnoCWzCYlOWYLG4C3yfTudZo5AcrTSgCTc= +github.com/r3labs/diff/v2 v2.14.2/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= @@ -110,6 +127,8 @@ github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GH github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= @@ -134,6 +153,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -182,9 +202,12 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -201,6 +224,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -211,6 +235,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/redis/redis.go b/redis/redis.go index 0ecd0b763c9c67a4dee09fd5df3aa0c57eafaa65..ff44a4bd1a0b3e5e4de01e21f0bb82cbe4c6cfb5 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -4,182 +4,148 @@ import ( "context" "encoding/json" "fmt" - "os" - "reflect" + "math" + "strings" "time" + "gitlab.com/uafrica/go-utils/errors" "gitlab.com/uafrica/go-utils/logs" + "github.com/go-redis/redis_rate/v9" + "github.com/go-redis/redis/v8" - "gitlab.com/uafrica/go-utils/errors" - "gitlab.com/uafrica/go-utils/string_utils" ) -type IRedis interface { - string_utils.KeyReader - Del(key string) error - SetJSON(key string, value interface{}) error - SetJSONIndefinitely(key string, value interface{}) error - SetJSONForDur(key string, value interface{}, dur time.Duration) error - GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool) - SetString(key string, value string) error - SetStringIndefinitely(key string, value string) error - SetStringForDur(key string, value string, dur time.Duration) error -} +var ctx = context.Background() -type redisWithContext struct { - context.Context - client *redis.Client +type ClientWithHelpers struct { + Client *redis.Client + Available bool } -func New(ctx context.Context) (IRedis, error) { - if globalClient == nil { - var err error - if globalClient, err = connect(); err != nil { - return redisWithContext{Context: ctx}, errors.Wrapf(err, "cannot connect to REDIS") - } +func NewClient(addr string) *ClientWithHelpers { + return &ClientWithHelpers{ + Client: redis.NewClient(&redis.Options{ + MaxRetries: 1, + DialTimeout: time.Duration(1) * time.Second, //So max 2 second wait + Addr: addr, + Password: "", // no password set + DB: 0, // use default Db + }), + Available: true, } - return redisWithContext{ - Context: ctx, - client: globalClient, - }, nil } -func (r redisWithContext) Del(key string) error { - if r.client == nil { +func (r ClientWithHelpers) IsConnected() bool { + return r.Client != nil && r.Available == true +} + +func (r ClientWithHelpers) DeleteByKey(key string) error { + if !r.IsConnected() { return errors.Errorf("REDIS disabled: cannot del key(%s)", key) } - _, err := r.client.Del(r.Context, key).Result() + _, err := r.Client.Del(ctx, key).Result() if err != nil { return errors.Wrapf(err, "failed to del key(%s)", key) } return nil } -//set JSON value for 24h -func (r redisWithContext) SetJSON(key string, value interface{}) error { - return r.SetJSONForDur(key, value, 24*time.Hour) -} - -func (r redisWithContext) SetJSONIndefinitely(key string, value interface{}) error { - return r.SetJSONForDur(key, value, 0) -} - -func (r redisWithContext) SetJSONForDur(key string, value interface{}, dur time.Duration) error { - if r.client == nil { - return errors.Errorf("REDIS disabled: cannot set JSON key(%s) = (%T)%v", key, value, value) +func (r ClientWithHelpers) DeleteByKeyPattern(pattern string) { + if !r.IsConnected() { + return } - valueStr, ok := value.(string) - if !ok { - jsonBytes, err := json.Marshal(value) + + iter := r.Client.Scan(ctx, 0, pattern, math.MaxInt64).Iterator() + for iter.Next(ctx) { + val := iter.Val() + err := r.Client.Del(ctx, val).Err() if err != nil { - return errors.Wrapf(err, "failed to JSON encode key(%s) = (%T)", key, value) + panic(err) } - valueStr = string(jsonBytes) } - if _, err := r.client.Set(r.Context, key, valueStr, dur).Result(); err != nil { - return errors.Wrapf(err, "failed to set JSON key(%s)", key) + if err := iter.Err(); err != nil { + panic(err) } - return nil } -//return: -// nil,nil if key is not defined -// nil,err if failed to get/determine if it exists, or failed to decode -// <value>,nil if found and decoded -func (r redisWithContext) GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool) { - if r.client == nil { - return nil, false +func (r ClientWithHelpers) SetObjectByKey(key string, object interface{}) { + if !r.IsConnected() { + return } - jsonValue, err := r.client.Get(r.Context, key).Result() + + jsonBytes, err := json.Marshal(object) if err != nil { - return nil, false + logs.ErrorWithMsg("Error marshalling object to Redis: %s", err) + return } - newValuePtr := reflect.New(valueType) - if err := json.Unmarshal([]byte(jsonValue), newValuePtr.Interface()); err != nil { - return nil, false - } - return newValuePtr.Elem().Interface(), true -} -func (r redisWithContext) SetString(key string, value string) error { - return r.SetStringForDur(key, value, 24*time.Hour) -} + _, err = r.Client.Set(ctx, key, string(jsonBytes), 24*time.Hour).Result() + if err != nil { + logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err) -func (r redisWithContext) SetStringIndefinitely(key string, value string) error { - return r.SetStringForDur(key, value, 0) + /* Prevent further calls in this execution from trying to connect and also timeout */ + if strings.HasSuffix(err.Error(), "i/o timeout") { + r.Available = false + } + } } -func (r redisWithContext) SetStringForDur(key string, value string, dur time.Duration) error { - if r.client == nil { - return errors.Errorf("REDIS disabled: cannot set key(%s) = (%T)%v", key, value, value) - } - if _, err := r.client.Set(r.Context, key, value, dur).Result(); err != nil { - return errors.Wrapf(err, "failed to set key(%s)", key) +func (r ClientWithHelpers) SetObjectByKeyIndefinitely(key string, object interface{}) { + if !r.IsConnected() { + return } - return nil -} -func (r redisWithContext) GetString(key string) (string, bool) { - if r.client == nil { - return "", false + jsonBytes, err := json.Marshal(object) + if err != nil { + logs.ErrorWithMsg("Error marshalling object to Redis", err) + return } - value, err := r.client.Get(r.Context, key).Result() - if err != nil { /* Actual error */ - if err != redis.Nil { /* other than Key does not exist */ - logs.LogErrorMessage(fmt.Sprintf("Error fetching redis key(%s): %+v", key), err) + + _, err = r.Client.Set(ctx, key, string(jsonBytes), 0).Result() + if err != nil { + logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err) + + /* Prevent further calls in this execution from trying to connect and also timeout */ + if strings.HasSuffix(err.Error(), "i/o timeout") { + r.Available = false } - return "", false } - return value, true + } -func (r redisWithContext) Keys(prefix string) []string { - if r.client == nil { - return nil +func (r ClientWithHelpers) GetValueByKey(key string) string { + if !r.IsConnected() { + return "" } - value, err := r.client.Keys(r.Context, prefix+"*").Result() - if err != nil { /* Actual error */ - if err != redis.Nil { /* other than no keys match */ - logs.LogErrorMessage(fmt.Sprintf("Error fetching redis keys(%s*): %+v", prefix), err) - } else { - logs.LogErrorMessage(fmt.Sprintf("Failed: %+v", err), err) + + jsonString, err := r.Client.Get(ctx, key).Result() + if err == redis.Nil { /* Key does not exist */ + return "" + } else if err != nil { /* Actual error */ + logs.Warn(fmt.Sprintf("Error fetching object from Redis for key: %s", key), err) + /* Prevent further calls in this execution from trying to connect and also timeout */ + if strings.HasSuffix(err.Error(), "i/o timeout") { + r.Available = false } - return nil //no matches } - return value + return jsonString } -//global connection to REDIS used in all context -var globalClient *redis.Client - -func connect() (*redis.Client, error) { - host := os.Getenv("REDIS_HOST") - if host == "false" { - return nil, errors.Errorf("REDIS_HOST=false") - } +func (r ClientWithHelpers) RateLimit(key string, limitFn func(int) redis_rate.Limit, limit int) (bool, error) { + limiter := redis_rate.NewLimiter(r.Client) + res, err := limiter.Allow(ctx, key, limitFn(limit)) + if err != nil { + logs.ErrorWithMsg(fmt.Sprintf("Redis Error rate limiting - %s", key), err) - port := os.Getenv("REDIS_PORT") - if os.Getenv("DEBUGGING") == "true" { - host = "host.docker.internal" - if os.Getenv("LOCAL") == "true" { - host = "localhost" - } - env := os.Getenv("ENVIRONMENT") - switch env { - case "dev": - port = "6380" - case "stage": - port = "6381" - case "prod": - port = "6383" + /* Prevent further calls in this execution from trying to connect and also timeout */ + if strings.HasSuffix(err.Error(), "i/o timeout") { + r.Available = false } + + return false, err + } else { + logs.Info("Rate limiter - %s : %t with used %d, remaining %d", key, res.Allowed == 1, limit-res.Remaining, res.Remaining) + return res.Allowed == 1, nil } - logs.Info("Using REDIS(%s:%s)", host, port) - globalClient = redis.NewClient(&redis.Options{ - Addr: host + ":" + port, - Password: "", // no password set - DB: 0, // use default DB - }) - return globalClient, nil -} //connect() +} diff --git a/redis/redis_test.go b/redis/redis_test.go deleted file mode 100644 index f6558892f934a0b37b169709662089301cac2e58..0000000000000000000000000000000000000000 --- a/redis/redis_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package redis_test - -import ( - "context" - "fmt" - "os" - "reflect" - "testing" - "time" - - "gitlab.com/uafrica/go-utils/redis" -) - -func TestString(t *testing.T) { - os.Setenv("REDIS_HOST", "localhost") - os.Setenv("REDIS_PORT", "6380") - ctx := context.Background() - r, err := redis.New(ctx) - if err != nil { - t.Fatalf("failed to create redis: %+v", err) - } - id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix()) - defer func() { - r.Del(id) - }() - - value := "abc123" - if err := r.SetString(id, value); err != nil { - t.Fatalf("failed to set: (%T) %+v", err, err) - } - - //get after set must return same value - if v, ok := r.GetString(id); !ok { - t.Fatalf("failed to get(%s)", id) - } else { - if v != value { - t.Fatalf("%s=%s != %s", id, v, value) - } - } - - //must be able to delete - if err := r.Del(id); err != nil { - t.Fatalf("failed to del(%s): %+v", id, err) - } - - //delete non-existing must also succeed - if err := r.Del(id); err != nil { - t.Fatalf("failed to del(%s) again: %+v", id, err) - } - - //get after delete must indicate not exist - if _, ok := r.GetString(id); ok { - t.Fatalf("got(%s) after delete", id) - } -} - -func TestJSON(t *testing.T) { - os.Setenv("REDIS_HOST", "localhost") - os.Setenv("REDIS_PORT", "6380") - ctx := context.Background() - r, err := redis.New(ctx) - if err != nil { - t.Fatalf("failed to create redis: %+v", err) - } - id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix()) - defer func() { - r.Del(id) - }() - - type Person struct { - Name string `json:"name"` - Surname string `json:"surname"` - Count int `json:"count"` - Dob time.Time `json:"dob"` - } - dob, err := time.Parse("2006-01-02", "1986-06-28") - if err != nil { - t.Fatalf("invalid dob: %+v", err) - } - value := Person{"Joe", "Blogs", 25, dob} - if err := r.SetJSON(id, value); err != nil { - t.Fatalf("failed to set: (%T) %+v", err, err) - } - - //get after set must return same value - if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); !ok { - t.Fatalf("failed to get(%s): %+v", id, err) - } else { - if v != value { - t.Fatalf("%s=%+v != %+v", id, v, value) - } - } - - //must be able to delete - if err := r.Del(id); err != nil { - t.Fatalf("failed to del(%s): %+v", id, err) - } - - //delete non-existing must also succeed - if err := r.Del(id); err != nil { - t.Fatalf("failed to del(%s) again: %+v", id, err) - } - - //get after delete must indicate not exist - if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); ok { - t.Fatalf("got(%s) after delete: %+v", id, v) - } -} - -func TestExp(t *testing.T) { - os.Setenv("REDIS_HOST", "localhost") - os.Setenv("REDIS_PORT", "6380") - ctx := context.Background() - r, err := redis.New(ctx) - if err != nil { - t.Fatalf("failed to create redis: %+v", err) - } - id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix()) - defer func() { - r.Del(id) - }() - - value := "abc123" - if err := r.SetStringForDur(id, value, time.Second); err != nil { - t.Fatalf("failed to set: (%T) %+v", err, err) - } - - //get after set must return same value - if v, ok := r.GetString(id); !ok { - t.Fatalf("failed to get(%s)", id) - } else { - if v != value { - t.Fatalf("%s=%s != %s", id, v, value) - } - } - - //wait 5 seconds - t.Logf("waiting 5seconds for key to expire...") - time.Sleep(time.Second * 5) - - //get after delete expire must fail - if _, ok := r.GetString(id); ok { - t.Fatalf("got(%s) after expiry", id) - } -} diff --git a/s3_utils/s3_utils.go b/s3_utils/s3_utils.go new file mode 100644 index 0000000000000000000000000000000000000000..f1c304962f1801d78bd681fd3661676bca7a0992 --- /dev/null +++ b/s3_utils/s3_utils.go @@ -0,0 +1,287 @@ +package s3_utils + +import ( + "bytes" + "encoding/binary" + "fmt" + "net/url" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/google/uuid" +) + +var s3Session *s3.S3 + +// S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request. +type S3UploadResponse struct { + URL string `json:"url"` + Filename string `json:"filename"` + Bucket string `json:"bucket"` + FileSize int `json:"file_size"` +} + +type MIMEType string + +const ( + // TypePDF defines the constant for the PDF MIME type. + MIMETypePDF MIMEType = "application/pdf" + + // TypeCSV defines the constant for the CSV MIME type. + MIMETypeCSV MIMEType = "text/csv" + + // TypeZIP defines the constant for the ZIP MIME type. + MIMETypeZIP MIMEType = "application/zip" + + // TypeJSON defines the constant for the JSON MIME type. + MIMETypeJSON MIMEType = "application/json" + + // MIMETypeText defines the constant for the Plain text MIME type. + MIMETypeText MIMEType = "text/plain" + + // MIMETypeImage defines the constant for the Image MIME type. + MIMETypeImage MIMEType = "image/*" + + // TypeDefault defines the constant for the default MIME type. + MIMETypeDefault MIMEType = "application/octet-stream" +) + +func Upload(data []byte, bucket, fileName string, metadata *map[string]*string, isDebug bool) (*s3.PutObjectOutput, error) { + mimeType := getTypeForFilename(fileName) + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + ContentType: aws.String(string(mimeType)), + Body: bytes.NewReader(data), + } + + if metadata != nil { + putInput.Metadata = *metadata + } + + response, err := getSession(isDebug).PutObject(putInput) + if err != nil { + return nil, err + } + + return response, nil +} + +func UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, isDebug bool) (string, error) { + if mimeType == "" { + mimeType = getTypeForFilename(fileName) + } + + expiry := time.Now().Add(24 * time.Hour) + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + ContentType: aws.String(string(mimeType)), + Body: bytes.NewReader(data), + Expires: &expiry, + } + + _, err := getSession(isDebug).PutObject(putInput) + if err != nil { + return "", err + } + + return GetSignedDownloadURL(bucket, fileName, 24*time.Hour, isDebug) +} + +// GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created. +func GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, isDebug bool) (string, error) { + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + } + getRequest, _ := getSession(isDebug).GetObjectRequest(getInput) + + return getRequest.Presign(duration) +} + +// UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse. +func UploadWithFileExtension(data []byte, bucket, filePrefix string, fileExt string, mimeType MIMEType, isDebug bool) (*S3UploadResponse, error) { + fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt) + + uploadUrl, err := UploadWithExpiry(data, bucket, fileName, mimeType, isDebug) + if err != nil { + return nil, err + } + + fileSizeInBytes := binary.Size(data) + + response := &S3UploadResponse{ + URL: uploadUrl, + Filename: fileName, + Bucket: bucket, + FileSize: fileSizeInBytes, + } + + return response, nil +} + +func getTypeForFilename(f string) MIMEType { + ext := strings.ToLower(path.Ext(f)) + + switch ext { + case "pdf": + return MIMETypePDF + case "csv": + return MIMETypeCSV + case "zip": + return MIMETypeZIP + case "txt": + return MIMETypeText + } + + return MIMETypeDefault +} + +func GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) { + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + } + getObjectOutput, err := getSession(isDebug).GetObject(getInput) + if err != nil { + return nil, err + } + return getObjectOutput, nil +} + +func GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) { + headObjectInput := &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + } + headObjectOutput, err := getSession(isDebug).HeadObject(headObjectInput) + if err != nil { + return nil, err + } + return headObjectOutput.Metadata, nil +} + +// MoveObjectBucketToBucket - Move object from one S3 bucket to another +func MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, isDebug bool) error { + + err := CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, isDebug) + if err != nil { + return err + } + + err = DeleteObjectFromBucket(sourceBucket, sourceFileName, isDebug) + if err != nil { + return err + } + + return nil +} + +// CopyObjectBucketToBucket - Copy an object from one S3 bucket to another +func CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, isDebug bool) error { + // copy the file + copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName) + copyObjectInput := &s3.CopyObjectInput{ + Bucket: aws.String(destinationBucket), //destination bucket + CopySource: aws.String(copySource), //source path (ie: myBucket/myFile.csv) + Key: aws.String(destinationFilename), //filename on destination + } + _, err := getSession(isDebug).CopyObject(copyObjectInput) + if err != nil { + return err + } + + // wait to see if the file copied successfully + err = getSession(isDebug).WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)}) + if err != nil { + return err + } + + return nil +} + +// DeleteObjectFromBucket - Delete an object from an S3 bucket +func DeleteObjectFromBucket(bucket string, fileName string, isDebug bool) error { + // delete the file + deleteObjectInput := &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fileName), + } + _, err := getSession(isDebug).DeleteObject(deleteObjectInput) + if err != nil { + return err + } + + // wait to see if the file deleted successfully + err = getSession(isDebug).WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(bucket), // the bucket we are deleting from + Key: aws.String(fileName), // the filename we are deleting + }) + if err != nil { + return err + } + + return nil +} + +func GetLogoURLFromFileName(fileName string) string { + logoUrl := "https://%s.s3.%s.amazonaws.com/logos/%s" + logoUrl = fmt.Sprintf(logoUrl, os.Getenv("S3_IMAGES_AND_NOTES_BUCKET"), os.Getenv("AWS_REGION"), fileName) + + return logoUrl +} + +func GetS3FileKey(fileName string, folder string) string { + // Trim leading and trailing slashes + fileName = strings.TrimLeft(fileName, "/") + fileName = strings.TrimRight(fileName, "/") + + folder = strings.TrimLeft(folder, "/") + folder = strings.TrimRight(folder, "/") + + return "/" + folder + "/" + fileName +} + +func ListObjects(bucketName string, maxKeys int64, function func(*s3.ListObjectsOutput, bool) bool, isDebug bool) error { + listObjectsInput := &s3.ListObjectsInput{ + Bucket: aws.String(bucketName), + MaxKeys: aws.Int64(maxKeys), + } + + err := getSession(isDebug).ListObjectsPages(listObjectsInput, function) + if err != nil { + return err + } + + return nil +} + +func getSession(isDebug bool) *s3.S3 { + if s3Session == nil { + env := os.Getenv("ENVIRONMENT") + options := session.Options{ + Config: aws.Config{ + Region: aws.String("af-south-1"), + }, + } + + if isDebug { + options.Profile = fmt.Sprintf("shiplogic-%s", env) + } + + sess, err := session.NewSessionWithOptions(options) + + if err != nil { + return nil + } + s3Session = s3.New(sess) + } + + return s3Session +} diff --git a/secrets_manager/secrets_manager.go b/secrets_manager/secrets_manager.go new file mode 100644 index 0000000000000000000000000000000000000000..34220c1e714639cca628550a5c5b63f676b074e3 --- /dev/null +++ b/secrets_manager/secrets_manager.go @@ -0,0 +1,124 @@ +package secrets_manager + +import ( + "encoding/base64" + "fmt" + "os" + + "gitlab.com/uafrica/go-utils/logs" + "gitlab.com/uafrica/go-utils/struct_utils" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/aws/aws-secretsmanager-caching-go/secretcache" +) + +type DatabaseCredentials struct { + Username string `json:"username"` + Password string `json:"password"` + Engine string `json:"engine"` + Host string `json:"host"` + Port int `json:"port"` + InstanceIdentifier string `json:"dbInstanceIdentifier"` +} + +var ( + secretCache, _ = secretcache.New() + secretManagerRegion = "af-south-1" +) + +func GetDatabaseCredentials(secretID string, isDebug bool) (DatabaseCredentials, error) { + secret, _ := getSecret(secretID, isDebug) + var credentials DatabaseCredentials + err := struct_utils.UnmarshalJSON([]byte(secret), &credentials) + if err != nil { + return DatabaseCredentials{}, err + } + return credentials, nil +} + +func getSecret(secretID string, isDebug bool) (string, string) { + cachedSecret, err := secretCache.GetSecretString(secretID) + if err != nil { + logs.Info("Failed to get secret key from cache") + } + if cachedSecret != "" { + return cachedSecret, "" + } + + awsSession := session.New() + + // Get local config + if isDebug && os.Getenv("ENVIRONMENT") != "" { + env := os.Getenv("ENVIRONMENT") + awsSession = session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Profile: fmt.Sprintf("shiplogic-%s", env), + })) + } + + // Create a Secrets Manager client + svc := secretsmanager.New(awsSession, aws.NewConfig().WithRegion(secretManagerRegion)) + + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(string(secretID)), + VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified + } + + // In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + // See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + + result, err := svc.GetSecretValue(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case secretsmanager.ErrCodeDecryptionFailure: + // Secrets Manager can't decrypt the protected secret text using the provided KMS key. + logs.Info(secretsmanager.ErrCodeDecryptionFailure, aerr.Error()) + + case secretsmanager.ErrCodeInternalServiceError: + // An error occurred on the server side. + logs.Info(secretsmanager.ErrCodeInternalServiceError, aerr.Error()) + + case secretsmanager.ErrCodeInvalidParameterException: + // You provided an invalid value for a parameter. + logs.Info(secretsmanager.ErrCodeInvalidParameterException, aerr.Error()) + + case secretsmanager.ErrCodeInvalidRequestException: + // You provided a parameter value that is not valid for the current state of the resource. + logs.Info(secretsmanager.ErrCodeInvalidRequestException, aerr.Error()) + + case secretsmanager.ErrCodeResourceNotFoundException: + // We can't find the resource that you asked for. + logs.Info("Can't find secret with ID: ", secretID) + logs.Info(secretsmanager.ErrCodeResourceNotFoundException, aerr.Error()) + default: + logs.Info(err.Error()) + } + } else { + // Print the error, cast err to awserr.Error to get the Code and + // Message from an error. + logs.Info(err.Error()) + } + return "", "" + } + + // Decrypts secret using the associated KMS CMK. + // Depending on whether the secret is a string or binary, one of these fields will be populated. + var secretString, decodedBinarySecret string + if result.SecretString != nil { + secretString = *result.SecretString + } else { + decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(result.SecretBinary))) + length, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, result.SecretBinary) + if err != nil { + logs.Info("Base64 Decode Error:", err) + return "", "" + } + decodedBinarySecret = string(decodedBinarySecretBytes[:length]) + } + + return secretString, decodedBinarySecret +} diff --git a/sqs/sqs.go b/sqs/sqs.go new file mode 100644 index 0000000000000000000000000000000000000000..6775da070c7931ed676c8a77ec6cd5c3892ed98e --- /dev/null +++ b/sqs/sqs.go @@ -0,0 +1,107 @@ +package sqs + +/*Package sqs provides a simple interface to send messages to AWS SQS*/ + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sqs" + "gitlab.com/uafrica/go-utils/logs" +) + +// Messenger sends an arbitrary message via SQS +type Messenger struct { + session *session.Session + service *sqs.SQS + queueURL string +} + +// NewSQSMessenger constructs a Messenger which sends messages to an SQS queue +// awsRegion - region that the queue was created +// awsQueue - name of the queue +// Note: Calling code needs SQS IAM permissions +func NewSQSMessenger(awsRegion, queueUrl string) (*Messenger, error) { + // Make an AWS session + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Region: aws.String(awsRegion), + }, + }) + + if err != nil { + return nil, err + } + + // Create SQS service + svc := sqs.New(sess) + + return &Messenger{ + session: sess, + service: svc, + queueURL: queueUrl, + }, nil +} + +// SendSQSMessage sends a message to the queue associated with the messenger +// headers - string message attributes of the SQS message (see AWS SQS documentation) +// body - body of the SQS message (see AWS SQS documentation) +func (m *Messenger) SendSQSMessage(headers map[string]string, body string, currentRequestID *string, headerKey string) (string, error) { + msgAttrs := make(map[string]*sqs.MessageAttributeValue) + + for key, val := range headers { + msgAttrs[key] = &sqs.MessageAttributeValue{ + DataType: aws.String("String"), + StringValue: aws.String(val), + } + } + + // Add request ID + if currentRequestID != nil { + msgAttrs[headerKey] = &sqs.MessageAttributeValue{ + DataType: aws.String("String"), + StringValue: aws.String(*currentRequestID), + } + } + + res, err := m.service.SendMessage(&sqs.SendMessageInput{ + MessageAttributes: msgAttrs, + MessageBody: aws.String(body), + QueueUrl: &m.queueURL, + }) + + if err != nil { + return "", err + } + + return *res.MessageId, err +} + +func SendSQSMessage(msgr *Messenger, region string, envQueueURLName string, objectToSend interface{}, currentRequestID *string, headerKey string) error { + if msgr == nil { + var err error + msgr, err = NewSQSMessenger(region, os.Getenv(envQueueURLName)) + if err != nil { + logs.ErrorWithMsg("Failed to create sqs messenger with envQueueURLName: "+envQueueURLName, err) + } + } + + jsonBytes, err := json.Marshal(objectToSend) + if err != nil { + logs.ErrorWithMsg("Failed to encode sqs event data", err) + return err + } + + headers := map[string]string{"Name": "dummy"} + msgID, err := msgr.SendSQSMessage(headers, string(jsonBytes), currentRequestID, headerKey) + if err != nil { + logs.ErrorWithMsg("Failed to send sqs event", err) + return err + } + + logs.Info(fmt.Sprintf("Sent SQS message to %s with ID %s", envQueueURLName, msgID)) + return nil +}