From b20650e10ed2037f76f9171c1a746beb80f9d26c Mon Sep 17 00:00:00 2001
From: Cornelius Rautenbach <cornel.rautenbach@gmail.com>
Date: Thu, 25 Nov 2021 15:39:19 +0200
Subject: [PATCH] Audit; Redis; S3; Secrets manager; SQS; Eina!
---
audit/audit.go | 183 ++++++++++++++++++
cognito/cognito.go | 135 ++++++++++++++
cognito/words.go | 104 +++++++++++
go.mod | 13 +-
go.sum | 37 +++-
redis/redis.go | 224 ++++++++++------------
redis/redis_test.go | 145 ---------------
s3_utils/s3_utils.go | 287 +++++++++++++++++++++++++++++
secrets_manager/secrets_manager.go | 124 +++++++++++++
sqs/sqs.go | 107 +++++++++++
10 files changed, 1077 insertions(+), 282 deletions(-)
create mode 100644 audit/audit.go
create mode 100644 cognito/cognito.go
create mode 100644 cognito/words.go
delete mode 100644 redis/redis_test.go
create mode 100644 s3_utils/s3_utils.go
create mode 100644 secrets_manager/secrets_manager.go
create mode 100644 sqs/sqs.go
diff --git a/audit/audit.go b/audit/audit.go
new file mode 100644
index 0000000..6d29688
--- /dev/null
+++ b/audit/audit.go
@@ -0,0 +1,183 @@
+package audit
+
+import (
+ "reflect"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/r3labs/diff/v2"
+ "gitlab.com/uafrica/go-utils/reflection"
+ "gitlab.com/uafrica/go-utils/string_utils"
+)
+
+type Event struct {
+ ID int64 `json:"id"`
+ ObjectID string `json:"object_id"`
+ Type string `json:"type"`
+ ProviderID int64 `json:"provider_id"`
+ Source string `json:"source"`
+ Timestamp time.Time `json:"timestamp"`
+ Change map[string]interface{} `json:"change"`
+}
+
+type FieldChange struct {
+ From interface{} `json:"change_from"`
+ To interface{} `json:"change_to"`
+}
+
+func GetAuditEvent(original interface{}, new interface{}, username *string) (Event, error) {
+ changelog, err := diff.Diff(original, new)
+ if err != nil {
+ return Event{}, err
+ }
+
+ changes := map[string]interface{}{}
+ for _, change := range changelog {
+
+ if len(change.Path) == 1 {
+ // Root object change
+ field := ToSnakeCase(change.Path[0])
+ changes[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ } else if len(change.Path) == 2 {
+ // Child object changed
+ // ["Account", "ID"]
+ // 0 = Object
+ // 1 = field
+
+ objectKey := ToSnakeCase(change.Path[0])
+ field := ToSnakeCase(change.Path[1])
+
+ existingObject, present := changes[objectKey]
+ if present {
+ if object, ok := existingObject.(map[string]interface{}); ok {
+ object[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ changes[objectKey] = object
+ }
+ } else {
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = fieldChange
+ }
+
+ } else if len(change.Path) == 3 {
+ // Array of objects
+ // ["Parcel", "0", "ActualWeight"]
+ // 0 = Object
+ // 1 = Index of object
+ // 2 = field
+
+ objectKey := ToSnakeCase(change.Path[0])
+ indexString := change.Path[1]
+ index, _ := string_utils.StringToInt64(indexString)
+ field := ToSnakeCase(change.Path[2])
+
+ arrayObject, present := changes[objectKey]
+ if present {
+ if arrayOfObjects, ok := arrayObject.([]map[string]interface{}); ok {
+ if len(arrayOfObjects) > int(index) {
+ // Add field to existing object in array
+ object := arrayOfObjects[index]
+ object[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ } else {
+ // new object, append to existing array
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = append(arrayOfObjects, fieldChange)
+ }
+
+ }
+ } else {
+ // Create array of objects
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = []map[string]interface{}{
+ fieldChange,
+ }
+ }
+ }
+ }
+
+ objectID := getIntValue(original, "ID")
+
+ if objectID == 0 {
+ objectID = getIntValue(new, "ID")
+ }
+
+ objectIDString := string_utils.Int64ToString(objectID)
+
+ if objectIDString == "0" {
+ objectIDString = getStringValue(original, "Username")
+ }
+
+ if objectIDString == "" {
+ objectIDString = getStringValue(new, "Username")
+ }
+
+ providerID := getIntValue(original, "ProviderID")
+ if providerID == 0 {
+ providerID = getIntValue(new, "ProviderID")
+ }
+
+ auditEvent := Event{
+ ObjectID: objectIDString,
+ ProviderID: providerID,
+ Source: "SYSTEM",
+ Timestamp: time.Now(),
+ Change: changes,
+ }
+ if username != nil {
+ auditEvent.Source = *username
+ }
+ return auditEvent, nil
+}
+
+var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
+var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
+
+func ToSnakeCase(str string) string {
+ snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
+ snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
+ return strings.ToLower(snake)
+}
+
+func getIntValue(object interface{}, key string) int64 {
+ structValue := reflect.ValueOf(object)
+ if structValue.Kind() == reflect.Struct {
+ field := structValue.FieldByName(key)
+ id := reflection.GetInt64Value(field)
+ return id
+ }
+ return 0
+}
+
+func getStringValue(object interface{}, key string) string {
+ structValue := reflect.ValueOf(object)
+ if structValue.Kind() == reflect.Struct {
+ field := structValue.FieldByName(key)
+ id := reflection.GetStringValue(field)
+ return id
+ }
+ return ""
+}
diff --git a/cognito/cognito.go b/cognito/cognito.go
new file mode 100644
index 0000000..afae2c9
--- /dev/null
+++ b/cognito/cognito.go
@@ -0,0 +1,135 @@
+package cognito
+
+import (
+ "fmt"
+ "math/rand"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/logs"
+
+ "github.com/aws/aws-lambda-go/events"
+ "github.com/aws/aws-sdk-go/service/cognitoidentityprovider"
+)
+
+var CognitoService *cognitoidentityprovider.CognitoIdentityProvider
+
+// ------------------------------------------------------------------------------------------------
+// Groups
+// ------------------------------------------------------------------------------------------------
+
+func AddCognitoUserToGroup(username string, pool string, group string) error {
+ groupInput := cognitoidentityprovider.AdminAddUserToGroupInput{
+ GroupName: &group,
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ groupOutput, err := CognitoService.AdminAddUserToGroup(&groupInput)
+ logs.Info("groupOutput", groupOutput)
+ return err
+}
+
+func RemoveCognitoUserFromGroup(username string, pool string, group string) error {
+ groupInput := cognitoidentityprovider.AdminRemoveUserFromGroupInput{
+ GroupName: &group,
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ groupOutput, err := CognitoService.AdminRemoveUserFromGroup(&groupInput)
+ logs.Info("groupOutput", groupOutput)
+ return err
+}
+
+// ------------------------------------------------------------------------------------------------
+// Users
+// ------------------------------------------------------------------------------------------------
+
+func GetCognitoUser(pool string, username string) (*cognitoidentityprovider.AdminGetUserOutput, error) {
+ userInput := cognitoidentityprovider.AdminGetUserInput{
+ UserPoolId: &pool,
+ Username: &username,
+ }
+
+ userOutput, err := CognitoService.AdminGetUser(&userInput)
+ if err != nil {
+ return nil, err
+ }
+ return userOutput, nil
+}
+
+// Deletes a user from its cognito user pool
+func DeleteCognitoUser(userPoolId string, username string) error {
+ input := cognitoidentityprovider.AdminDeleteUserInput{
+ UserPoolId: &userPoolId,
+ Username: &username,
+ }
+ output, err := CognitoService.AdminDeleteUser(&input)
+ logs.Info("output", output)
+ return err
+}
+
+func ResetUserPassword(pool string, username string) (*cognitoidentityprovider.AdminResetUserPasswordOutput, error) {
+ input := cognitoidentityprovider.AdminResetUserPasswordInput{
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ output, err := CognitoService.AdminResetUserPassword(&input)
+ logs.Info("output", output)
+ return output, err
+}
+
+func SetUserPassword(pool string, username string, password string) (*cognitoidentityprovider.AdminSetUserPasswordOutput, error) {
+ setPermanently := true
+ input := cognitoidentityprovider.AdminSetUserPasswordInput{
+ UserPoolId: &pool,
+ Username: &username,
+ Password: &password,
+ Permanent: &setPermanently,
+ }
+ output, err := CognitoService.AdminSetUserPassword(&input)
+ logs.Info("output", output)
+ return output, err
+}
+
+// FOR API LOGS
+
+func DetermineAuthType(identity events.APIGatewayRequestIdentity) *string {
+ result := "cognito"
+ if identity.CognitoAuthenticationType == "" {
+ result = "iam"
+ }
+
+ return &result
+}
+
+func GetAuthUsername(identity events.APIGatewayRequestIdentity) string {
+ if identity.CognitoAuthenticationProvider != "" {
+ split := strings.Split(identity.CognitoAuthenticationProvider, ":")
+ return split[len(split)-1]
+ }
+
+ // IAM
+ split := strings.Split(identity.UserArn, ":user/")
+ return split[len(split)-1]
+}
+
+// Create a pseudorandom password consisting of two three-letter words and two digits
+func RandomPassword() string {
+ i := rand.Intn(100)
+ var j int
+ for {
+ j = rand.Intn(100)
+ if j != i {
+ break
+ }
+ }
+ return fmt.Sprintf("%s%s%s", words[i], words[j], RandomDigitString(2))
+}
+
+// Create a pseudorandom string of digits (0-9) with specified length
+func RandomDigitString(len int) string {
+ var str strings.Builder
+ for i := 0; i < len; i++ {
+ fmt.Fprintf(&str, "%v", rand.Intn(10))
+ }
+ return str.String()
+}
diff --git a/cognito/words.go b/cognito/words.go
new file mode 100644
index 0000000..72c46fb
--- /dev/null
+++ b/cognito/words.go
@@ -0,0 +1,104 @@
+package cognito
+
+var words = []string{
+ "act",
+ "add",
+ "age",
+ "air",
+ "ant",
+ "ark",
+ "arm",
+ "art",
+ "ash",
+ "bad",
+ "bag",
+ "bar",
+ "bat",
+ "bay",
+ "bed",
+ "bee",
+ "big",
+ "box",
+ "bus",
+ "bye",
+ "can",
+ "car",
+ "cat",
+ "con",
+ "cow",
+ "cup",
+ "day",
+ "dog",
+ "ear",
+ "end",
+ "eye",
+ "far",
+ "fly",
+ "fox",
+ "fry",
+ "fun",
+ "gap",
+ "gym",
+ "hat",
+ "hip",
+ "hop",
+ "ice",
+ "ink",
+ "jam",
+ "jar",
+ "joy",
+ "key",
+ "kit",
+ "lab",
+ "law",
+ "log",
+ "low",
+ "man",
+ "max",
+ "may",
+ "net",
+ "nut",
+ "oil",
+ "one",
+ "out",
+ "owl",
+ "own",
+ "pen",
+ "pet",
+ "pie",
+ "pig",
+ "pin",
+ "pro",
+ "ram",
+ "rap",
+ "ray",
+ "red",
+ "row",
+ "sea",
+ "see",
+ "sew",
+ "sit",
+ "six",
+ "sky",
+ "son",
+ "spy",
+ "tan",
+ "tax",
+ "tea",
+ "ted",
+ "tee",
+ "ten",
+ "the",
+ "tin",
+ "toe",
+ "top",
+ "toy",
+ "tub",
+ "two",
+ "way",
+ "wig",
+ "yes",
+ "you",
+ "zip",
+ "zoo",
+}
diff --git a/go.mod b/go.mod
index 3678562..059784e 100644
--- a/go.mod
+++ b/go.mod
@@ -5,31 +5,40 @@ go 1.17
require (
github.com/MindscapeHQ/raygun4go v1.1.1
github.com/aws/aws-lambda-go v1.26.0
+ github.com/aws/aws-sdk-go v1.42.12
+ github.com/aws/aws-secretsmanager-caching-go v1.1.0
github.com/go-pg/pg/v10 v10.10.6
- github.com/go-redis/redis/v8 v8.11.3
+ github.com/go-redis/redis/v8 v8.11.4
+ github.com/go-redis/redis_rate/v9 v9.1.2
github.com/google/uuid v1.3.0
github.com/opensearch-project/opensearch-go v1.0.0
github.com/pkg/errors v0.9.1
+ github.com/r3labs/diff/v2 v2.14.2
github.com/sirupsen/logrus v1.8.1
github.com/thoas/go-funk v0.9.1
golang.org/x/text v0.3.7
)
require (
- github.com/cespare/xxhash/v2 v2.1.1 // indirect
+ github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-errors/errors v1.4.1 // indirect
github.com/go-pg/zerochecker v0.2.0 // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/vmihailenco/bufpool v0.1.11 // indirect
+ github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect
+ google.golang.org/appengine v1.6.6 // indirect
+ google.golang.org/protobuf v1.26.0 // indirect
mellium.im/sasl v0.2.1 // indirect
)
diff --git a/go.sum b/go.sum
index 3630870..8e11358 100644
--- a/go.sum
+++ b/go.sum
@@ -4,9 +4,14 @@ github.com/MindscapeHQ/raygun4go v1.1.1 h1:fk3Uknv9kQxUIwL3mywwHQRyfq3PaR9lE/e40
github.com/MindscapeHQ/raygun4go v1.1.1/go.mod h1:NW0eWi2Qs00ZcctO6owrVMY+h2HxzJVgQGDrTj2ysw4=
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
+github.com/aws/aws-sdk-go v1.19.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA=
+github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-secretsmanager-caching-go v1.1.0 h1:vcV94XGJ9KouXKYBTMqgrBw96Tae8JKLmoUZ5SbaXNo=
+github.com/aws/aws-secretsmanager-caching-go v1.1.0/go.mod h1:wahQpJP1dZKMqjGFAjGCqilHkTlN0zReGWocPLbXmxg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -26,12 +31,15 @@ github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso=
github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg=
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
-github.com/go-redis/redis/v8 v8.11.3 h1:GCjoYp8c+yQTJfc0n69iwSiHjvuAdruxl7elnZCxgt8=
-github.com/go-redis/redis/v8 v8.11.3/go.mod h1:xNJ9xDG09FsIPwh3bWdk+0oDWHbtF9rPN0F/oD9XeKc=
+github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
+github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
+github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ=
+github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -42,6 +50,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -49,6 +58,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
@@ -58,6 +68,11 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -77,8 +92,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
-github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/opensearch-project/opensearch-go v1.0.0 h1:8Gh7B7Un5BxuxWAgmzleEF7lpOtC71pCgPp7lKr3ca8=
github.com/opensearch-project/opensearch-go v1.0.0/go.mod h1:FrUl/52DBegRYvK7ISF278AXmjDV647lyTnsLGBR7J4=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
@@ -88,6 +103,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/r3labs/diff/v2 v2.14.2 h1:1HVhQKwg1YnoCWzCYlOWYLG4C3yfTudZo5AcrTSgCTc=
+github.com/r3labs/diff/v2 v2.14.2/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
@@ -110,6 +127,8 @@ github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GH
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
+github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
+github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc=
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
@@ -134,6 +153,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -182,9 +202,12 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
@@ -201,6 +224,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -211,6 +235,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/redis/redis.go b/redis/redis.go
index 0ecd0b7..ff44a4b 100644
--- a/redis/redis.go
+++ b/redis/redis.go
@@ -4,182 +4,148 @@ import (
"context"
"encoding/json"
"fmt"
- "os"
- "reflect"
+ "math"
+ "strings"
"time"
+ "gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logs"
+ "github.com/go-redis/redis_rate/v9"
+
"github.com/go-redis/redis/v8"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/string_utils"
)
-type IRedis interface {
- string_utils.KeyReader
- Del(key string) error
- SetJSON(key string, value interface{}) error
- SetJSONIndefinitely(key string, value interface{}) error
- SetJSONForDur(key string, value interface{}, dur time.Duration) error
- GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool)
- SetString(key string, value string) error
- SetStringIndefinitely(key string, value string) error
- SetStringForDur(key string, value string, dur time.Duration) error
-}
+var ctx = context.Background()
-type redisWithContext struct {
- context.Context
- client *redis.Client
+type ClientWithHelpers struct {
+ Client *redis.Client
+ Available bool
}
-func New(ctx context.Context) (IRedis, error) {
- if globalClient == nil {
- var err error
- if globalClient, err = connect(); err != nil {
- return redisWithContext{Context: ctx}, errors.Wrapf(err, "cannot connect to REDIS")
- }
+func NewClient(addr string) *ClientWithHelpers {
+ return &ClientWithHelpers{
+ Client: redis.NewClient(&redis.Options{
+ MaxRetries: 1,
+ DialTimeout: time.Duration(1) * time.Second, //So max 2 second wait
+ Addr: addr,
+ Password: "", // no password set
+ DB: 0, // use default Db
+ }),
+ Available: true,
}
- return redisWithContext{
- Context: ctx,
- client: globalClient,
- }, nil
}
-func (r redisWithContext) Del(key string) error {
- if r.client == nil {
+func (r ClientWithHelpers) IsConnected() bool {
+ return r.Client != nil && r.Available == true
+}
+
+func (r ClientWithHelpers) DeleteByKey(key string) error {
+ if !r.IsConnected() {
return errors.Errorf("REDIS disabled: cannot del key(%s)", key)
}
- _, err := r.client.Del(r.Context, key).Result()
+ _, err := r.Client.Del(ctx, key).Result()
if err != nil {
return errors.Wrapf(err, "failed to del key(%s)", key)
}
return nil
}
-//set JSON value for 24h
-func (r redisWithContext) SetJSON(key string, value interface{}) error {
- return r.SetJSONForDur(key, value, 24*time.Hour)
-}
-
-func (r redisWithContext) SetJSONIndefinitely(key string, value interface{}) error {
- return r.SetJSONForDur(key, value, 0)
-}
-
-func (r redisWithContext) SetJSONForDur(key string, value interface{}, dur time.Duration) error {
- if r.client == nil {
- return errors.Errorf("REDIS disabled: cannot set JSON key(%s) = (%T)%v", key, value, value)
+func (r ClientWithHelpers) DeleteByKeyPattern(pattern string) {
+ if !r.IsConnected() {
+ return
}
- valueStr, ok := value.(string)
- if !ok {
- jsonBytes, err := json.Marshal(value)
+
+ iter := r.Client.Scan(ctx, 0, pattern, math.MaxInt64).Iterator()
+ for iter.Next(ctx) {
+ val := iter.Val()
+ err := r.Client.Del(ctx, val).Err()
if err != nil {
- return errors.Wrapf(err, "failed to JSON encode key(%s) = (%T)", key, value)
+ panic(err)
}
- valueStr = string(jsonBytes)
}
- if _, err := r.client.Set(r.Context, key, valueStr, dur).Result(); err != nil {
- return errors.Wrapf(err, "failed to set JSON key(%s)", key)
+ if err := iter.Err(); err != nil {
+ panic(err)
}
- return nil
}
-//return:
-// nil,nil if key is not defined
-// nil,err if failed to get/determine if it exists, or failed to decode
-// <value>,nil if found and decoded
-func (r redisWithContext) GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool) {
- if r.client == nil {
- return nil, false
+func (r ClientWithHelpers) SetObjectByKey(key string, object interface{}) {
+ if !r.IsConnected() {
+ return
}
- jsonValue, err := r.client.Get(r.Context, key).Result()
+
+ jsonBytes, err := json.Marshal(object)
if err != nil {
- return nil, false
+ logs.ErrorWithMsg("Error marshalling object to Redis: %s", err)
+ return
}
- newValuePtr := reflect.New(valueType)
- if err := json.Unmarshal([]byte(jsonValue), newValuePtr.Interface()); err != nil {
- return nil, false
- }
- return newValuePtr.Elem().Interface(), true
-}
-func (r redisWithContext) SetString(key string, value string) error {
- return r.SetStringForDur(key, value, 24*time.Hour)
-}
+ _, err = r.Client.Set(ctx, key, string(jsonBytes), 24*time.Hour).Result()
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err)
-func (r redisWithContext) SetStringIndefinitely(key string, value string) error {
- return r.SetStringForDur(key, value, 0)
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
+ }
+ }
}
-func (r redisWithContext) SetStringForDur(key string, value string, dur time.Duration) error {
- if r.client == nil {
- return errors.Errorf("REDIS disabled: cannot set key(%s) = (%T)%v", key, value, value)
- }
- if _, err := r.client.Set(r.Context, key, value, dur).Result(); err != nil {
- return errors.Wrapf(err, "failed to set key(%s)", key)
+func (r ClientWithHelpers) SetObjectByKeyIndefinitely(key string, object interface{}) {
+ if !r.IsConnected() {
+ return
}
- return nil
-}
-func (r redisWithContext) GetString(key string) (string, bool) {
- if r.client == nil {
- return "", false
+ jsonBytes, err := json.Marshal(object)
+ if err != nil {
+ logs.ErrorWithMsg("Error marshalling object to Redis", err)
+ return
}
- value, err := r.client.Get(r.Context, key).Result()
- if err != nil { /* Actual error */
- if err != redis.Nil { /* other than Key does not exist */
- logs.LogErrorMessage(fmt.Sprintf("Error fetching redis key(%s): %+v", key), err)
+
+ _, err = r.Client.Set(ctx, key, string(jsonBytes), 0).Result()
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err)
+
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
- return "", false
}
- return value, true
+
}
-func (r redisWithContext) Keys(prefix string) []string {
- if r.client == nil {
- return nil
+func (r ClientWithHelpers) GetValueByKey(key string) string {
+ if !r.IsConnected() {
+ return ""
}
- value, err := r.client.Keys(r.Context, prefix+"*").Result()
- if err != nil { /* Actual error */
- if err != redis.Nil { /* other than no keys match */
- logs.LogErrorMessage(fmt.Sprintf("Error fetching redis keys(%s*): %+v", prefix), err)
- } else {
- logs.LogErrorMessage(fmt.Sprintf("Failed: %+v", err), err)
+
+ jsonString, err := r.Client.Get(ctx, key).Result()
+ if err == redis.Nil { /* Key does not exist */
+ return ""
+ } else if err != nil { /* Actual error */
+ logs.Warn(fmt.Sprintf("Error fetching object from Redis for key: %s", key), err)
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
- return nil //no matches
}
- return value
+ return jsonString
}
-//global connection to REDIS used in all context
-var globalClient *redis.Client
-
-func connect() (*redis.Client, error) {
- host := os.Getenv("REDIS_HOST")
- if host == "false" {
- return nil, errors.Errorf("REDIS_HOST=false")
- }
+func (r ClientWithHelpers) RateLimit(key string, limitFn func(int) redis_rate.Limit, limit int) (bool, error) {
+ limiter := redis_rate.NewLimiter(r.Client)
+ res, err := limiter.Allow(ctx, key, limitFn(limit))
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Redis Error rate limiting - %s", key), err)
- port := os.Getenv("REDIS_PORT")
- if os.Getenv("DEBUGGING") == "true" {
- host = "host.docker.internal"
- if os.Getenv("LOCAL") == "true" {
- host = "localhost"
- }
- env := os.Getenv("ENVIRONMENT")
- switch env {
- case "dev":
- port = "6380"
- case "stage":
- port = "6381"
- case "prod":
- port = "6383"
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
+
+ return false, err
+ } else {
+ logs.Info("Rate limiter - %s : %t with used %d, remaining %d", key, res.Allowed == 1, limit-res.Remaining, res.Remaining)
+ return res.Allowed == 1, nil
}
- logs.Info("Using REDIS(%s:%s)", host, port)
- globalClient = redis.NewClient(&redis.Options{
- Addr: host + ":" + port,
- Password: "", // no password set
- DB: 0, // use default DB
- })
- return globalClient, nil
-} //connect()
+}
diff --git a/redis/redis_test.go b/redis/redis_test.go
deleted file mode 100644
index f655889..0000000
--- a/redis/redis_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-package redis_test
-
-import (
- "context"
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "gitlab.com/uafrica/go-utils/redis"
-)
-
-func TestString(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- value := "abc123"
- if err := r.SetString(id, value); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetString(id); !ok {
- t.Fatalf("failed to get(%s)", id)
- } else {
- if v != value {
- t.Fatalf("%s=%s != %s", id, v, value)
- }
- }
-
- //must be able to delete
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s): %+v", id, err)
- }
-
- //delete non-existing must also succeed
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s) again: %+v", id, err)
- }
-
- //get after delete must indicate not exist
- if _, ok := r.GetString(id); ok {
- t.Fatalf("got(%s) after delete", id)
- }
-}
-
-func TestJSON(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- type Person struct {
- Name string `json:"name"`
- Surname string `json:"surname"`
- Count int `json:"count"`
- Dob time.Time `json:"dob"`
- }
- dob, err := time.Parse("2006-01-02", "1986-06-28")
- if err != nil {
- t.Fatalf("invalid dob: %+v", err)
- }
- value := Person{"Joe", "Blogs", 25, dob}
- if err := r.SetJSON(id, value); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); !ok {
- t.Fatalf("failed to get(%s): %+v", id, err)
- } else {
- if v != value {
- t.Fatalf("%s=%+v != %+v", id, v, value)
- }
- }
-
- //must be able to delete
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s): %+v", id, err)
- }
-
- //delete non-existing must also succeed
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s) again: %+v", id, err)
- }
-
- //get after delete must indicate not exist
- if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); ok {
- t.Fatalf("got(%s) after delete: %+v", id, v)
- }
-}
-
-func TestExp(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- value := "abc123"
- if err := r.SetStringForDur(id, value, time.Second); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetString(id); !ok {
- t.Fatalf("failed to get(%s)", id)
- } else {
- if v != value {
- t.Fatalf("%s=%s != %s", id, v, value)
- }
- }
-
- //wait 5 seconds
- t.Logf("waiting 5seconds for key to expire...")
- time.Sleep(time.Second * 5)
-
- //get after delete expire must fail
- if _, ok := r.GetString(id); ok {
- t.Fatalf("got(%s) after expiry", id)
- }
-}
diff --git a/s3_utils/s3_utils.go b/s3_utils/s3_utils.go
new file mode 100644
index 0000000..f1c3049
--- /dev/null
+++ b/s3_utils/s3_utils.go
@@ -0,0 +1,287 @@
+package s3_utils
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/google/uuid"
+)
+
+var s3Session *s3.S3
+
+// S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request.
+type S3UploadResponse struct {
+ URL string `json:"url"`
+ Filename string `json:"filename"`
+ Bucket string `json:"bucket"`
+ FileSize int `json:"file_size"`
+}
+
+type MIMEType string
+
+const (
+ // TypePDF defines the constant for the PDF MIME type.
+ MIMETypePDF MIMEType = "application/pdf"
+
+ // TypeCSV defines the constant for the CSV MIME type.
+ MIMETypeCSV MIMEType = "text/csv"
+
+ // TypeZIP defines the constant for the ZIP MIME type.
+ MIMETypeZIP MIMEType = "application/zip"
+
+ // TypeJSON defines the constant for the JSON MIME type.
+ MIMETypeJSON MIMEType = "application/json"
+
+ // MIMETypeText defines the constant for the Plain text MIME type.
+ MIMETypeText MIMEType = "text/plain"
+
+ // MIMETypeImage defines the constant for the Image MIME type.
+ MIMETypeImage MIMEType = "image/*"
+
+ // TypeDefault defines the constant for the default MIME type.
+ MIMETypeDefault MIMEType = "application/octet-stream"
+)
+
+func Upload(data []byte, bucket, fileName string, metadata *map[string]*string, isDebug bool) (*s3.PutObjectOutput, error) {
+ mimeType := getTypeForFilename(fileName)
+ putInput := &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ ContentType: aws.String(string(mimeType)),
+ Body: bytes.NewReader(data),
+ }
+
+ if metadata != nil {
+ putInput.Metadata = *metadata
+ }
+
+ response, err := getSession(isDebug).PutObject(putInput)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+func UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, isDebug bool) (string, error) {
+ if mimeType == "" {
+ mimeType = getTypeForFilename(fileName)
+ }
+
+ expiry := time.Now().Add(24 * time.Hour)
+ putInput := &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ ContentType: aws.String(string(mimeType)),
+ Body: bytes.NewReader(data),
+ Expires: &expiry,
+ }
+
+ _, err := getSession(isDebug).PutObject(putInput)
+ if err != nil {
+ return "", err
+ }
+
+ return GetSignedDownloadURL(bucket, fileName, 24*time.Hour, isDebug)
+}
+
+// GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created.
+func GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, isDebug bool) (string, error) {
+ getInput := &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ getRequest, _ := getSession(isDebug).GetObjectRequest(getInput)
+
+ return getRequest.Presign(duration)
+}
+
+// UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse.
+func UploadWithFileExtension(data []byte, bucket, filePrefix string, fileExt string, mimeType MIMEType, isDebug bool) (*S3UploadResponse, error) {
+ fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt)
+
+ uploadUrl, err := UploadWithExpiry(data, bucket, fileName, mimeType, isDebug)
+ if err != nil {
+ return nil, err
+ }
+
+ fileSizeInBytes := binary.Size(data)
+
+ response := &S3UploadResponse{
+ URL: uploadUrl,
+ Filename: fileName,
+ Bucket: bucket,
+ FileSize: fileSizeInBytes,
+ }
+
+ return response, nil
+}
+
+func getTypeForFilename(f string) MIMEType {
+ ext := strings.ToLower(path.Ext(f))
+
+ switch ext {
+ case "pdf":
+ return MIMETypePDF
+ case "csv":
+ return MIMETypeCSV
+ case "zip":
+ return MIMETypeZIP
+ case "txt":
+ return MIMETypeText
+ }
+
+ return MIMETypeDefault
+}
+
+func GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) {
+ getInput := &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ getObjectOutput, err := getSession(isDebug).GetObject(getInput)
+ if err != nil {
+ return nil, err
+ }
+ return getObjectOutput, nil
+}
+
+func GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) {
+ headObjectInput := &s3.HeadObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ headObjectOutput, err := getSession(isDebug).HeadObject(headObjectInput)
+ if err != nil {
+ return nil, err
+ }
+ return headObjectOutput.Metadata, nil
+}
+
+// MoveObjectBucketToBucket - Move object from one S3 bucket to another
+func MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, isDebug bool) error {
+
+ err := CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, isDebug)
+ if err != nil {
+ return err
+ }
+
+ err = DeleteObjectFromBucket(sourceBucket, sourceFileName, isDebug)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CopyObjectBucketToBucket - Copy an object from one S3 bucket to another
+func CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, isDebug bool) error {
+ // copy the file
+ copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName)
+ copyObjectInput := &s3.CopyObjectInput{
+ Bucket: aws.String(destinationBucket), //destination bucket
+ CopySource: aws.String(copySource), //source path (ie: myBucket/myFile.csv)
+ Key: aws.String(destinationFilename), //filename on destination
+ }
+ _, err := getSession(isDebug).CopyObject(copyObjectInput)
+ if err != nil {
+ return err
+ }
+
+ // wait to see if the file copied successfully
+ err = getSession(isDebug).WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// DeleteObjectFromBucket - Delete an object from an S3 bucket
+func DeleteObjectFromBucket(bucket string, fileName string, isDebug bool) error {
+ // delete the file
+ deleteObjectInput := &s3.DeleteObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ _, err := getSession(isDebug).DeleteObject(deleteObjectInput)
+ if err != nil {
+ return err
+ }
+
+ // wait to see if the file deleted successfully
+ err = getSession(isDebug).WaitUntilObjectNotExists(&s3.HeadObjectInput{
+ Bucket: aws.String(bucket), // the bucket we are deleting from
+ Key: aws.String(fileName), // the filename we are deleting
+ })
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func GetLogoURLFromFileName(fileName string) string {
+ logoUrl := "https://%s.s3.%s.amazonaws.com/logos/%s"
+ logoUrl = fmt.Sprintf(logoUrl, os.Getenv("S3_IMAGES_AND_NOTES_BUCKET"), os.Getenv("AWS_REGION"), fileName)
+
+ return logoUrl
+}
+
+func GetS3FileKey(fileName string, folder string) string {
+ // Trim leading and trailing slashes
+ fileName = strings.TrimLeft(fileName, "/")
+ fileName = strings.TrimRight(fileName, "/")
+
+ folder = strings.TrimLeft(folder, "/")
+ folder = strings.TrimRight(folder, "/")
+
+ return "/" + folder + "/" + fileName
+}
+
+func ListObjects(bucketName string, maxKeys int64, function func(*s3.ListObjectsOutput, bool) bool, isDebug bool) error {
+ listObjectsInput := &s3.ListObjectsInput{
+ Bucket: aws.String(bucketName),
+ MaxKeys: aws.Int64(maxKeys),
+ }
+
+ err := getSession(isDebug).ListObjectsPages(listObjectsInput, function)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func getSession(isDebug bool) *s3.S3 {
+ if s3Session == nil {
+ env := os.Getenv("ENVIRONMENT")
+ options := session.Options{
+ Config: aws.Config{
+ Region: aws.String("af-south-1"),
+ },
+ }
+
+ if isDebug {
+ options.Profile = fmt.Sprintf("shiplogic-%s", env)
+ }
+
+ sess, err := session.NewSessionWithOptions(options)
+
+ if err != nil {
+ return nil
+ }
+ s3Session = s3.New(sess)
+ }
+
+ return s3Session
+}
diff --git a/secrets_manager/secrets_manager.go b/secrets_manager/secrets_manager.go
new file mode 100644
index 0000000..34220c1
--- /dev/null
+++ b/secrets_manager/secrets_manager.go
@@ -0,0 +1,124 @@
+package secrets_manager
+
+import (
+ "encoding/base64"
+ "fmt"
+ "os"
+
+ "gitlab.com/uafrica/go-utils/logs"
+ "gitlab.com/uafrica/go-utils/struct_utils"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/secretsmanager"
+ "github.com/aws/aws-secretsmanager-caching-go/secretcache"
+)
+
+type DatabaseCredentials struct {
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Engine string `json:"engine"`
+ Host string `json:"host"`
+ Port int `json:"port"`
+ InstanceIdentifier string `json:"dbInstanceIdentifier"`
+}
+
+var (
+ secretCache, _ = secretcache.New()
+ secretManagerRegion = "af-south-1"
+)
+
+func GetDatabaseCredentials(secretID string, isDebug bool) (DatabaseCredentials, error) {
+ secret, _ := getSecret(secretID, isDebug)
+ var credentials DatabaseCredentials
+ err := struct_utils.UnmarshalJSON([]byte(secret), &credentials)
+ if err != nil {
+ return DatabaseCredentials{}, err
+ }
+ return credentials, nil
+}
+
+func getSecret(secretID string, isDebug bool) (string, string) {
+ cachedSecret, err := secretCache.GetSecretString(secretID)
+ if err != nil {
+ logs.Info("Failed to get secret key from cache")
+ }
+ if cachedSecret != "" {
+ return cachedSecret, ""
+ }
+
+ awsSession := session.New()
+
+ // Get local config
+ if isDebug && os.Getenv("ENVIRONMENT") != "" {
+ env := os.Getenv("ENVIRONMENT")
+ awsSession = session.Must(session.NewSessionWithOptions(session.Options{
+ SharedConfigState: session.SharedConfigEnable,
+ Profile: fmt.Sprintf("shiplogic-%s", env),
+ }))
+ }
+
+ // Create a Secrets Manager client
+ svc := secretsmanager.New(awsSession, aws.NewConfig().WithRegion(secretManagerRegion))
+
+ input := &secretsmanager.GetSecretValueInput{
+ SecretId: aws.String(string(secretID)),
+ VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified
+ }
+
+ // In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
+ // See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+
+ result, err := svc.GetSecretValue(input)
+ if err != nil {
+ if aerr, ok := err.(awserr.Error); ok {
+ switch aerr.Code() {
+ case secretsmanager.ErrCodeDecryptionFailure:
+ // Secrets Manager can't decrypt the protected secret text using the provided KMS key.
+ logs.Info(secretsmanager.ErrCodeDecryptionFailure, aerr.Error())
+
+ case secretsmanager.ErrCodeInternalServiceError:
+ // An error occurred on the server side.
+ logs.Info(secretsmanager.ErrCodeInternalServiceError, aerr.Error())
+
+ case secretsmanager.ErrCodeInvalidParameterException:
+ // You provided an invalid value for a parameter.
+ logs.Info(secretsmanager.ErrCodeInvalidParameterException, aerr.Error())
+
+ case secretsmanager.ErrCodeInvalidRequestException:
+ // You provided a parameter value that is not valid for the current state of the resource.
+ logs.Info(secretsmanager.ErrCodeInvalidRequestException, aerr.Error())
+
+ case secretsmanager.ErrCodeResourceNotFoundException:
+ // We can't find the resource that you asked for.
+ logs.Info("Can't find secret with ID: ", secretID)
+ logs.Info(secretsmanager.ErrCodeResourceNotFoundException, aerr.Error())
+ default:
+ logs.Info(err.Error())
+ }
+ } else {
+ // Print the error, cast err to awserr.Error to get the Code and
+ // Message from an error.
+ logs.Info(err.Error())
+ }
+ return "", ""
+ }
+
+ // Decrypts secret using the associated KMS CMK.
+ // Depending on whether the secret is a string or binary, one of these fields will be populated.
+ var secretString, decodedBinarySecret string
+ if result.SecretString != nil {
+ secretString = *result.SecretString
+ } else {
+ decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(result.SecretBinary)))
+ length, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, result.SecretBinary)
+ if err != nil {
+ logs.Info("Base64 Decode Error:", err)
+ return "", ""
+ }
+ decodedBinarySecret = string(decodedBinarySecretBytes[:length])
+ }
+
+ return secretString, decodedBinarySecret
+}
diff --git a/sqs/sqs.go b/sqs/sqs.go
new file mode 100644
index 0000000..6775da0
--- /dev/null
+++ b/sqs/sqs.go
@@ -0,0 +1,107 @@
+package sqs
+
+/*Package sqs provides a simple interface to send messages to AWS SQS*/
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/sqs"
+ "gitlab.com/uafrica/go-utils/logs"
+)
+
+// Messenger sends an arbitrary message via SQS
+type Messenger struct {
+ session *session.Session
+ service *sqs.SQS
+ queueURL string
+}
+
+// NewSQSMessenger constructs a Messenger which sends messages to an SQS queue
+// awsRegion - region that the queue was created
+// awsQueue - name of the queue
+// Note: Calling code needs SQS IAM permissions
+func NewSQSMessenger(awsRegion, queueUrl string) (*Messenger, error) {
+ // Make an AWS session
+ sess, err := session.NewSessionWithOptions(session.Options{
+ Config: aws.Config{
+ Region: aws.String(awsRegion),
+ },
+ })
+
+ if err != nil {
+ return nil, err
+ }
+
+ // Create SQS service
+ svc := sqs.New(sess)
+
+ return &Messenger{
+ session: sess,
+ service: svc,
+ queueURL: queueUrl,
+ }, nil
+}
+
+// SendSQSMessage sends a message to the queue associated with the messenger
+// headers - string message attributes of the SQS message (see AWS SQS documentation)
+// body - body of the SQS message (see AWS SQS documentation)
+func (m *Messenger) SendSQSMessage(headers map[string]string, body string, currentRequestID *string, headerKey string) (string, error) {
+ msgAttrs := make(map[string]*sqs.MessageAttributeValue)
+
+ for key, val := range headers {
+ msgAttrs[key] = &sqs.MessageAttributeValue{
+ DataType: aws.String("String"),
+ StringValue: aws.String(val),
+ }
+ }
+
+ // Add request ID
+ if currentRequestID != nil {
+ msgAttrs[headerKey] = &sqs.MessageAttributeValue{
+ DataType: aws.String("String"),
+ StringValue: aws.String(*currentRequestID),
+ }
+ }
+
+ res, err := m.service.SendMessage(&sqs.SendMessageInput{
+ MessageAttributes: msgAttrs,
+ MessageBody: aws.String(body),
+ QueueUrl: &m.queueURL,
+ })
+
+ if err != nil {
+ return "", err
+ }
+
+ return *res.MessageId, err
+}
+
+func SendSQSMessage(msgr *Messenger, region string, envQueueURLName string, objectToSend interface{}, currentRequestID *string, headerKey string) error {
+ if msgr == nil {
+ var err error
+ msgr, err = NewSQSMessenger(region, os.Getenv(envQueueURLName))
+ if err != nil {
+ logs.ErrorWithMsg("Failed to create sqs messenger with envQueueURLName: "+envQueueURLName, err)
+ }
+ }
+
+ jsonBytes, err := json.Marshal(objectToSend)
+ if err != nil {
+ logs.ErrorWithMsg("Failed to encode sqs event data", err)
+ return err
+ }
+
+ headers := map[string]string{"Name": "dummy"}
+ msgID, err := msgr.SendSQSMessage(headers, string(jsonBytes), currentRequestID, headerKey)
+ if err != nil {
+ logs.ErrorWithMsg("Failed to send sqs event", err)
+ return err
+ }
+
+ logs.Info(fmt.Sprintf("Sent SQS message to %s with ID %s", envQueueURLName, msgID))
+ return nil
+}
--
GitLab