diff --git a/address_utils/address_utils.go b/address_utils/address_utils.go
index 2cf7ff2afbcac70af2aa20d9e1fe54e0de4b9b69..b90563d946ed467dffef1b931563f8140c647c3c 100644
--- a/address_utils/address_utils.go
+++ b/address_utils/address_utils.go
@@ -3,21 +3,32 @@ package address_utils
import (
"crypto/md5"
"fmt"
+ "math"
"regexp"
"strings"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils"
)
+var southAfricaVariations = []string{
+ "ZA",
+ "South Africa",
+ "Suid-Afrika",
+ "Suid Afrika",
+ "Iningizimu Afrika",
+ "Mzantsi Afrika",
+ "Afrika Boroa",
+ "Africa Kusini",
+}
+
type Province struct {
- Code string
- Names []string
+ Names []string
+ DisplayName string
}
-// Provinces largely follows the ISO standard: https://en.wikipedia.org/wiki/ISO_3166-2:ZA
-var Provinces = []Province{
- {
- Code: "EC",
+var Provinces = map[string]Province{
+ "EC": {
+ DisplayName: "Eastern Cape",
Names: []string{
"Eastern Cape",
"Eastern-Cape",
@@ -33,8 +44,8 @@ var Provinces = []Province{
"Mpumalanga-Koloni",
},
},
- {
- Code: "FS",
+ "FS": {
+ DisplayName: "Free State",
Names: []string{
"Free State",
"Freestate",
@@ -48,18 +59,21 @@ var Provinces = []Province{
"Freyisitata",
},
},
- {
- Code: "GP",
+ "GP": {
+ DisplayName: "Gauteng",
Names: []string{
+ "GT",
"Gauteng",
"iGauteng",
"Kgauteng",
"Rhawuti",
},
},
- {
- Code: "KZN",
+ "KZN": {
+ DisplayName: "KwaZulu-Natal",
Names: []string{
+ "NT",
+ "NL",
"KwaZulu-Natal",
"KwaZulu Natal",
"iKwaZulu-Natal",
@@ -70,22 +84,23 @@ var Provinces = []Province{
"KwaZulu-Natala",
},
},
- {
- Code: "LP",
+ "LP": {
+ DisplayName: "Limpopo",
Names: []string{
+ "NP",
"Limpopo",
"Vhembe",
},
},
- {
- Code: "MP",
+ "MP": {
+ DisplayName: "Mpumalanga",
Names: []string{
"Mpumalanga",
"iMpumalanga",
},
},
- {
- Code: "NC",
+ "NC": {
+ DisplayName: "Northern Cape",
Names: []string{
"Northern Cape",
"Northern-Cape",
@@ -102,8 +117,8 @@ var Provinces = []Province{
"Nyakatho-Koloni",
},
},
- {
- Code: "NW",
+ "NW": {
+ DisplayName: "North West",
Names: []string{
"North West",
"North-West",
@@ -119,8 +134,8 @@ var Provinces = []Province{
"Nyakatho-Ntshonalanga",
},
},
- {
- Code: "WC",
+ "WC": {
+ DisplayName: "Western Cape",
Names: []string{
"Western Cape",
"Western-Cape",
@@ -212,8 +227,6 @@ func stripUnwantedCharacters(s string) string {
func CleanZone(countryToClean, zoneToClean *string) (newCountry, newZone *string) {
newCountry = countryToClean
- southAfricaVariations := []string{"ZA", "South Africa", "Suid-Afrika", "Suid Afrika", "Iningizimu Afrika", "Mzantsi Afrika", "Afrika Boroa", "Africa Kusini"}
-
for _, southAfricaVariation := range southAfricaVariations {
if countryToClean == nil || len(*countryToClean) == 0 || strings.ToLower(*countryToClean) == strings.ToLower(southAfricaVariation) {
defaultCountry := "South Africa"
@@ -224,20 +237,10 @@ func CleanZone(countryToClean, zoneToClean *string) (newCountry, newZone *string
if *newCountry == "South Africa" && zoneToClean != nil {
zone := *zoneToClean
- if zone == "GT" {
- // Gauteng - GT should be GP for Google
- zone = "GP"
- } else if zone == "NT" || zone == "NL" {
- // KZN - NT and NL should be KZN for Google
- zone = "KZN"
- } else if zone == "NP" {
- // Limpopo - NP should be LP for Google
- zone = "LP"
- }
- for _, province := range Provinces {
+ for provinceCode, province := range Provinces {
for _, name := range province.Names {
- zone = string_utils.ReplaceCaseInsensitive(zone, name, province.Code)
+ zone = string_utils.ReplaceCaseInsensitive(zone, name, provinceCode)
}
}
@@ -248,11 +251,48 @@ func CleanZone(countryToClean, zoneToClean *string) (newCountry, newZone *string
}
func IsProvince(address string) bool {
- for _, province := range Provinces {
- if strings.ToLower(address) == strings.ToLower(fmt.Sprintf("%v, South Africa", province)) {
+ for provinceCode, province := range Provinces {
+ if strings.ToLower(address) == strings.ToLower(fmt.Sprintf("%v, South Africa", provinceCode)) {
+ return true
+ }
+ if strings.ToLower(address) == strings.ToLower(fmt.Sprintf("%v, South Africa", province.DisplayName)) {
return true
}
+ for _, provinceName := range province.Names {
+ if strings.ToLower(address) == strings.ToLower(fmt.Sprintf("%v, South Africa", provinceName)) {
+ return true
+ }
+ }
}
return false
}
+
+func GetZoneDisplayName(zone string) string {
+
+ if province, ok := Provinces[zone]; ok {
+ return province.DisplayName
+ }
+
+ return zone
+}
+
+func GetDistanceInKmBetweenCoordinates(lat1 float64, lng1 float64, lat2 float64, lng2 float64) float64 {
+ radlat1 := float64(math.Pi * lat1 / 180)
+ radlat2 := float64(math.Pi * lat2 / 180)
+
+ theta := float64(lng1 - lng2)
+ radtheta := float64(math.Pi * theta / 180)
+
+ dist := math.Sin(radlat1)*math.Sin(radlat2) + math.Cos(radlat1)*math.Cos(radlat2)*math.Cos(radtheta)
+ if dist > 1 {
+ dist = 1
+ }
+
+ dist = math.Acos(dist)
+ dist = dist * 180 / math.Pi
+ dist = dist * 60 * 1.1515
+
+ // Return in kilometers
+ return dist * 1.609344
+}
diff --git a/address_utils/address_utils_test.go b/address_utils/address_utils_test.go
index 330513062229b050a694b8c5d39456ef799f7d4a..09f0b4366a6deb6682b21706c4d7763f7a7db4dc 100644
--- a/address_utils/address_utils_test.go
+++ b/address_utils/address_utils_test.go
@@ -25,6 +25,14 @@ func TestIsProvince(t *testing.T) {
name: "IsProvince",
args: args{address: "North West, South Africa"},
want: true,
+ }, {
+ name: "IsProvince2",
+ args: args{address: "KwaZulu Natal, South Africa"},
+ want: true,
+ }, {
+ name: "IsProvince3",
+ args: args{address: "KZN, South Africa"},
+ want: true,
}, {
name: "IsNotProvince",
args: args{address: "22 Kruis Street, Potchefstroom, Potchefstroom, 2531, GP, ZA"},
@@ -39,3 +47,38 @@ func TestIsProvince(t *testing.T) {
})
}
}
+
+func TestZoneDisplayName(t *testing.T) {
+ type args struct {
+ zone string
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ }{{
+ name: "IsValidZone",
+ args: args{zone: "LP"},
+ want: "Limpopo",
+ }, {
+ name: "IsValidZone2",
+ args: args{zone: "KZN"},
+ want: "KwaZulu-Natal",
+ }, {
+ name: "IsNotValidZone",
+ args: args{zone: "invalidF"},
+ want: "invalidF",
+ }, {
+ name: "IsNotValidZone 2",
+ args: args{zone: "NP"},
+ want: "NP",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := GetZoneDisplayName(tt.args.zone); got != tt.want {
+ t.Errorf("GetZoneDisplayName() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/api_logs/api-logs.go b/api_logs/api-logs.go
index 33043db418d4c60e4ba4a9501ad8089e90d7710f..bfd4ad1f3eb87f61e0b5e90e13dd0ceba0a92927 100644
--- a/api_logs/api-logs.go
+++ b/api_logs/api-logs.go
@@ -2,7 +2,7 @@ package api_logs
import (
"github.com/thoas/go-funk"
- "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/number_utils"
"net/url"
"strings"
"time"
@@ -41,7 +41,7 @@ func GenerateIncomingAPILog(startTime time.Time, requestID *string, claim map[st
if accountID == 0 {
if accountIDParam, ok := req.QueryStringParameters["account_id"]; ok {
- if i64, err := string_utils.StringToInt64(accountIDParam); err == nil && i64 > 0 {
+ if i64, err := number_utils.StringToInt64(accountIDParam); err == nil && i64 > 0 {
accountID = i64
}
}
@@ -49,7 +49,7 @@ func GenerateIncomingAPILog(startTime time.Time, requestID *string, claim map[st
if providerID == 0 {
if providerIDParam, ok := req.QueryStringParameters["provider_id"]; ok {
- if i64, err := string_utils.StringToInt64(providerIDParam); err == nil && i64 > 0 {
+ if i64, err := number_utils.StringToInt64(providerIDParam); err == nil && i64 > 0 {
providerID = i64
}
}
@@ -60,6 +60,14 @@ func GenerateIncomingAPILog(startTime time.Time, requestID *string, claim map[st
typeString = "webhook-incoming"
}
+ // Remove the API key in the header
+ if req.Headers["authorization"] != "" {
+ req.Headers["authorization"] = "***"
+ }
+ if req.Headers["Authorization"] != "" {
+ req.Headers["Authorization"] = "***"
+ }
+
apiLog := ApiLog{
StartTime: startTime,
EndTime: endTime,
diff --git a/api_responses/api_responses.go b/api_responses/api_responses.go
index 4774cc3513f5ad0a641487362475a99afc653340..b728cb5f0ba6bac519729c12c1b5edde4006c800 100644
--- a/api_responses/api_responses.go
+++ b/api_responses/api_responses.go
@@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/map_utils"
"net/http"
"regexp"
"strconv"
@@ -46,14 +47,14 @@ func Error(err error, msg string, statusCode int) (events.APIGatewayProxyRespons
if err != nil {
return events.APIGatewayProxyResponse{
StatusCode: statusCode,
- Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
Body: "{ \"error\": \"" + http.StatusText(http.StatusInternalServerError) + "\"}",
}, nil
}
return events.APIGatewayProxyResponse{
StatusCode: statusCode,
- Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
Body: string(bodyBytes),
}, errors.New(msg)
}
@@ -136,7 +137,7 @@ func DatabaseServerError(err error, msg string) (events.APIGatewayProxyResponse,
if marshalError != nil {
return events.APIGatewayProxyResponse{
StatusCode: statusCode,
- Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
Body: "{ \"error\": \"" + http.StatusText(http.StatusInternalServerError) + "\"}",
}, nil
}
@@ -148,7 +149,7 @@ func DatabaseServerError(err error, msg string) (events.APIGatewayProxyResponse,
return events.APIGatewayProxyResponse{
StatusCode: statusCode,
- Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
Body: string(bodyBytes),
}, err
}
@@ -230,7 +231,7 @@ func ClientError(status int, message string) (events.APIGatewayProxyResponse, er
return events.APIGatewayProxyResponse{
StatusCode: status,
- Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
Body: string(b),
}, errors.New(message)
}
diff --git a/audit/audit.go b/audit/audit.go
index 8c8052427a1dd43b660e3ded7293b422500e0001..580513581cf6a07d8e1e2f08a9fdcff7fa8e6294 100644
--- a/audit/audit.go
+++ b/audit/audit.go
@@ -3,7 +3,8 @@ package audit
import (
"encoding/json"
"fmt"
- "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/number_utils"
"reflect"
"regexp"
"strconv"
@@ -77,7 +78,7 @@ func GetChanges(original interface{}, new interface{}) (map[string]interface{},
continue
}
- index, _ := string_utils.StringToInt64(indexString)
+ index, _ := number_utils.StringToInt64(indexString)
field := ToSnakeCase(change.Path[2])
if len(change.Path) == 5 && string_utils.IsNumericString(change.Path[3]) {
diff --git a/auth/api_key.go b/auth/api_key.go
new file mode 100644
index 0000000000000000000000000000000000000000..739dfc08b1d972ec7b2058d0e9358047d3610794
--- /dev/null
+++ b/auth/api_key.go
@@ -0,0 +1,38 @@
+package auth
+
+import (
+ "github.com/google/uuid"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/utils"
+ "strings"
+)
+
+// GenerateNewApiKey generates a 32 character API key. If the build environment is dev or stage the key will start with
+// "dev_" or "stage_" respectively.
+func GenerateNewApiKey() string {
+ uniqueKey := uuid.New().String()
+ uniqueKey = strings.ReplaceAll(uniqueKey, "-", "")
+
+ env := utils.GetEnv("ENVIRONMENT", "")
+ if env == "dev" || env == "stage" {
+ uniqueKey = env + "_" + uniqueKey
+ }
+
+ return uniqueKey
+}
+
+// GetApiKeyFromHeaders checks if a bearer token is passed as part of the Authorization header and returns that key.
+// NOTE: This function is deprecated. It simply calls the more generic GetBearerTokenFromHeaders.
+func GetApiKeyFromHeaders(headers map[string]string) string {
+ return GetBearerTokenFromHeaders(headers)
+}
+
+// MaskAPIKey masks an API key in the form "abc***xyz"
+func MaskAPIKey(key string) string {
+ keyRunes := []rune(key)
+ keyLength := len(keyRunes)
+ if keyLength > 6 {
+ return string(keyRunes[:3]) + "***" + string(keyRunes[keyLength-3:])
+ }
+ // This shouldn't happen, but if we don't have more than 6 characters, mask in the form "***z"
+ return "***" + string(keyRunes[keyLength-1])
+}
diff --git a/auth/common.go b/auth/common.go
new file mode 100644
index 0000000000000000000000000000000000000000..c9a37eccab18bc1bd1be9661cba91183037e93ad
--- /dev/null
+++ b/auth/common.go
@@ -0,0 +1,32 @@
+package auth
+
+import (
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/handler_utils"
+ "golang.org/x/crypto/bcrypt"
+ "strings"
+)
+
+// GetBearerTokenFromHeaders checks if a bearer token is passed as part of the Authorization header and returns that key
+func GetBearerTokenFromHeaders(headers map[string]string) string {
+ headerValue := handler_utils.FindHeaderValue(headers, "authorization")
+ if strings.HasPrefix(strings.ToLower(headerValue), "bearer ") {
+ headerValues := strings.Split(headerValue, " ")
+ return strings.TrimSpace(headerValues[1])
+ }
+ return ""
+}
+
+// HashPassword returns a hashed version of the provided password.
+func HashPassword(password string) (string, error) {
+ encryptedBytes, err := bcrypt.GenerateFromPassword([]byte(password), 14)
+ if err != nil {
+ return "", err
+ }
+ return string(encryptedBytes), nil
+}
+
+// PasswordIsCorrect checks whether the password is correct by validating it against the hashed password.
+func PasswordIsCorrect(password string, hashedPassword string) bool {
+ err := bcrypt.CompareHashAndPassword([]byte(hashedPassword), []byte(password))
+ return err == nil
+}
diff --git a/auth/jwt.go b/auth/jwt.go
new file mode 100644
index 0000000000000000000000000000000000000000..55824e254b55e6e2eb4d97d8db95b18e158877b5
--- /dev/null
+++ b/auth/jwt.go
@@ -0,0 +1,158 @@
+package auth
+
+import (
+ "encoding/json"
+ "github.com/golang-jwt/jwt/v4"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/date_utils"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "net/http"
+ "time"
+)
+
+type JsonWebToken struct {
+ UserID string `json:"user_id"`
+ ProviderID int64 `json:"provider_id,omitempty"`
+ ExpiryDate time.Time `json:"expiry_date"`
+}
+
+// GenerateJWTWithSessionToken first signs the session token with the secret key, then takes the payload and generates a
+// signed JWT using the resulting signed session token
+func GenerateJWTWithSessionToken(payload JsonWebToken, secretKey string, sessionToken string) (string, error) {
+ signedSessionToken := SignSessionTokenWithKey(secretKey, sessionToken)
+ return GenerateJWT(payload, signedSessionToken)
+}
+
+// GenerateJWT takes the payload and generates a signed JWT using the provided secret
+func GenerateJWT(payload JsonWebToken, secretKey string) (string, error) {
+ // Convert the JsonWebToken to a map[string]interface{}
+ tokenBytes, err := json.Marshal(payload)
+ if err != nil {
+ return "", err
+ }
+
+ tokenMap := make(map[string]interface{})
+ err = json.Unmarshal(tokenBytes, &tokenMap)
+ if err != nil {
+ return "", err
+ }
+
+ // Create the signed token
+ token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims(tokenMap))
+ tokenString, err := token.SignedString([]byte(secretKey))
+ if err != nil {
+ return "", err
+ }
+
+ return tokenString, nil
+}
+
+func getJsonWebTokenFromTokenClaims(token *jwt.Token, checkValidity bool) (JsonWebToken, error) {
+ if token == nil {
+ return JsonWebToken{}, errors.Error("could not get token from token string")
+ }
+
+ claims, ok := token.Claims.(jwt.MapClaims)
+ if !ok || (checkValidity && token.Valid == false) {
+ return JsonWebToken{}, errors.Error("invalid token")
+ }
+
+ // Convert the MapClaims to a JsonWebToken
+ claimsBytes, err := json.Marshal(claims)
+ if err != nil {
+ return JsonWebToken{}, err
+ }
+
+ var jsonWebToken JsonWebToken
+ err = json.Unmarshal(claimsBytes, &jsonWebToken)
+ if err != nil {
+ return JsonWebToken{}, err
+ }
+
+ return jsonWebToken, nil
+}
+
+// ValidateJWTWithSessionToken first signs the session token using the secret key, then parses the JWT and validates
+// that it is signed correctly
+func ValidateJWTWithSessionToken(jsonWebTokenString string, secretKey string, sessionToken string) (JsonWebToken, error) {
+ // Sign the session token with the secret key - this prevents the JWT from being used by other sessions
+ signedSecret := SignSessionTokenWithKey(secretKey, sessionToken)
+ return ValidateJWT(jsonWebTokenString, signedSecret)
+}
+
+// ValidateJWT parses the JWT and validates that it is signed correctly
+func ValidateJWT(jsonWebTokenString string, secretKey string) (JsonWebToken, error) {
+ // Validate the
+ token, err := jwt.Parse(jsonWebTokenString, func(token *jwt.Token) (interface{}, error) {
+ // Validate the signing method
+ if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+ return nil, errors.Errorf("unexpected signing method: %v", token.Header["alg"])
+ }
+
+ return []byte(secretKey), nil
+ })
+ if err != nil {
+ return JsonWebToken{}, err
+ }
+
+ jsonWebToken, err := getJsonWebTokenFromTokenClaims(token, true)
+ if err != nil {
+ return JsonWebToken{}, err
+ }
+
+ // Validate the expiry date
+ if jsonWebToken.ExpiryDate.Before(date_utils.CurrentDate()) {
+ return jsonWebToken, errors.Error("token has expired")
+ }
+
+ return jsonWebToken, nil
+}
+
+// LoginWithPassword checks that the provided password is correct. If the password is correct, a signed JWT is returned
+// using the provided encryption key.
+func LoginWithPassword(password string, hashedPassword string, jsonWebToken JsonWebToken, jwtEncryptionKey string) (string, error) {
+ if PasswordIsCorrect(password, hashedPassword) {
+ return GenerateJWT(jsonWebToken, jwtEncryptionKey)
+ }
+
+ return "", errors.HTTPWithMsg(http.StatusBadRequest, "password is incorrect")
+}
+
+// LoginSessionWithPassword checks that the provided password is correct. If the password is correct, the session token
+// is signed using the secret key, and a JWT is returned using the signed session token
+func LoginSessionWithPassword(password string, hashedPassword string, jsonWebToken JsonWebToken, secretKey string, sessionToken string) (string, error) {
+ if PasswordIsCorrect(password, hashedPassword) {
+ return GenerateJWTWithSessionToken(jsonWebToken, secretKey, sessionToken)
+ }
+
+ return "", errors.HTTPWithMsg(http.StatusBadRequest, "password is incorrect")
+}
+
+// GetUserIDFromJWTWithoutValidation gets the userID from the jsonWebTokenString without validating the signature.
+// Successful execution of this function DOES NOT indicate that the JWT is valid in any way.
+func GetUserIDFromJWTWithoutValidation(jsonWebTokenString string) string {
+ token, _, err := jwt.NewParser().ParseUnverified(jsonWebTokenString, jwt.MapClaims{})
+ if err != nil {
+ return ""
+ }
+
+ jsonWebToken, err := getJsonWebTokenFromTokenClaims(token, false)
+ if err != nil {
+ return ""
+ }
+ return jsonWebToken.UserID
+}
+
+// GetUserAndProviderIDFromJWTWithoutValidation gets the userID and providerID from the jsonWebTokenString without validating the
+// signature. Successful execution of this function DOES NOT indicate that the JWT is valid in any way.
+func GetUserAndProviderIDFromJWTWithoutValidation(jsonWebTokenString string) (string, int64) {
+ token, _, err := jwt.NewParser().ParseUnverified(jsonWebTokenString, jwt.MapClaims{})
+ if err != nil {
+ return "", 0
+ }
+
+ jsonWebToken, err := getJsonWebTokenFromTokenClaims(token, false)
+ if err != nil {
+ return "", 0
+ }
+ return jsonWebToken.UserID, jsonWebToken.ProviderID
+}
diff --git a/auth/jwt_test.go b/auth/jwt_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..8fe270b56a56dda2482976d7bd89f5e2dbf1146a
--- /dev/null
+++ b/auth/jwt_test.go
@@ -0,0 +1,68 @@
+package auth
+
+import (
+ "fmt"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/date_utils"
+ "testing"
+ "time"
+)
+
+func TestJWT(t *testing.T) {
+ secret := []byte("THISISTHESECRETKEY")
+
+ jwToken := JsonWebToken{
+ UserID: "janoadmin",
+ Password: "Password123",
+ ProviderID: 1,
+ ExpiryDate: date_utils.CurrentDate().Add(time.Second * 1),
+ }
+
+ // Validate before expire
+ fmt.Println("TEST VALID JWT")
+ jwTokenString, err := GenerateJWT(jwToken, secret)
+ if err != nil {
+ fmt.Println("!!!!FAIL", err.Error())
+ t.FailNow()
+ }
+ validatedToken, err := ValidateJWT(jwTokenString, secret)
+ if err != nil {
+ fmt.Println("!!!!FAIL", err.Error())
+ t.Fail()
+ } else {
+ fmt.Println(" SUCCESS", validatedToken)
+ }
+
+ // Validate invalid signature
+ fmt.Println("TEST INVALID JWT SIGNATURE")
+ jwTokenString, err = GenerateJWT(jwToken, []byte("INCORRECTSECRETVALUE"))
+ if err != nil {
+ fmt.Println("!!!!FAIL", err.Error())
+ t.FailNow()
+ }
+ validatedToken, err = ValidateJWT(jwTokenString, secret)
+ if err == nil {
+ fmt.Println("!!!!FAIL")
+ t.Fail()
+ } else {
+ fmt.Println(" SUCCESS", err.Error())
+ }
+
+ time.Sleep(time.Second * 1)
+
+ // Validate after expire
+ fmt.Println("TEST EXPIRED JWT")
+ jwTokenString, err = GenerateJWT(jwToken, secret)
+ if err != nil {
+ fmt.Println("!!!!FAIL", err.Error())
+ t.FailNow()
+ }
+ validatedToken, err = ValidateJWT(jwTokenString, secret)
+ if err == nil {
+ fmt.Println("!!!!FAIL")
+ t.Fail()
+ } else {
+ fmt.Println(" SUCCESS", err.Error())
+ }
+
+ fmt.Println("Done!")
+}
diff --git a/auth/session.go b/auth/session.go
new file mode 100644
index 0000000000000000000000000000000000000000..da9035ee441b41bdec52f9d60339248b6b109a47
--- /dev/null
+++ b/auth/session.go
@@ -0,0 +1,110 @@
+package auth
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "github.com/aws/aws-lambda-go/events"
+ "github.com/google/uuid"
+ "github.com/thoas/go-funk"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/date_utils"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/handler_utils"
+ "time"
+)
+
+type SessionToken struct {
+ IP string `json:"ip"`
+ UserAgent string `json:"user_agent"`
+ TimeCreated time.Time `json:"time_created"`
+ Token string `json:"session_token"`
+}
+
+// SignSessionTokenWithKey signs the session token string using the secret.
+func SignSessionTokenWithKey(secretKey string, sessionTokenString string) string {
+ signer := hmac.New(sha256.New, []byte(secretKey))
+ signer.Write([]byte(sessionTokenString))
+ return hex.EncodeToString(signer.Sum(nil))
+}
+
+// GetSessionTokenString creates a unique session token string from the API request.
+func GetSessionTokenString(request events.APIGatewayProxyRequest) (string, error) {
+ sessionToken := SessionToken{
+ IP: request.RequestContext.Identity.SourceIP,
+ UserAgent: handler_utils.FindHeaderValue(request.Headers, "user-agent"),
+ TimeCreated: date_utils.CurrentDate().Round(time.Second),
+ Token: uuid.New().String(),
+ }
+ sessionTokenBytes, err := json.Marshal(sessionToken)
+ return string(sessionTokenBytes), err
+}
+
+// GetSignedSessionTokenString creates a unique session token string from the API request and signs it using the secret.
+func GetSignedSessionTokenString(request events.APIGatewayProxyRequest, secretKey string) (string, error) {
+ sessionTokenBytes, err := GetSessionTokenString(request)
+ if err != nil {
+ return "", err
+ }
+
+ return SignSessionTokenWithKey(secretKey, sessionTokenBytes), nil
+}
+
+// ValidateJWTWithSessionTokens attempts to validate the JWT string by signing each session token using the secret, and
+// using the resulting signed session token to validate the JWT. If the JWT can be validated using a session token, the
+// JsonWebToken is returned, otherwise nil is returned. If the JWT is expired, nil is returned along with the session token.
+func ValidateJWTWithSessionTokens(jsonWebTokenString string, secretKey string, sessionTokens []string) (validJsonWebToken *JsonWebToken, expiredSessionToken *string) {
+ // Test each session token to find one that is valid
+ for _, sessionToken := range sessionTokens {
+ jsonWebToken, err := ValidateJWTWithSessionToken(jsonWebTokenString, secretKey, sessionToken)
+ if err == nil {
+ return &jsonWebToken, nil
+ } else if err.Error() == "token has expired" {
+ return nil, &sessionToken
+ }
+ }
+
+ return nil, nil
+}
+
+// FindAndRemoveCurrentSessionToken attempts to validate the JWT string by signing each session token using the secret,
+// and using the resulting signed session token to validate the JWT. If the JWT is successfully validated with one of
+// the session tokens, the session token is removed from the slice, otherwise the original session token
+// slice is returned.
+func FindAndRemoveCurrentSessionToken(jsonWebTokenString string, secretKey string, sessionTokens []string) (string, []string) {
+ // Test each session token to find one that is valid
+ for _, sessionToken := range sessionTokens {
+ _, err := ValidateJWTWithSessionToken(jsonWebTokenString, secretKey, sessionToken)
+ if err == nil {
+ // Remove this session token from the slice
+ updatedSessionTokens := funk.FilterString(sessionTokens, func(token string) bool {
+ return token != sessionToken
+ })
+ return sessionToken, updatedSessionTokens
+ }
+ }
+
+ return "", sessionTokens
+}
+
+// RemoveInvalidatedAndOldSessionTokens removes the provided invalidated session token, and checks the age of the
+// other session tokens and removes the ones that are older than the provided age.
+func RemoveInvalidatedAndOldSessionTokens(sessionTokens []string, invalidatedSessionToken string, age time.Duration) []string {
+ ageDurationAgo := date_utils.CurrentDate().Add(-1 * age)
+ validTokens := funk.FilterString(sessionTokens, func(sessionTokenString string) bool {
+ // Always remove the invalidated session token
+ if sessionTokenString == invalidatedSessionToken {
+ return false
+ }
+
+ var sessionToken SessionToken
+ err := json.Unmarshal([]byte(sessionTokenString), &sessionToken)
+ if err != nil {
+ // If we can't unmarshal the token then it is not valid
+ return false
+ }
+
+ // Keep the token if it hasn't expired yet
+ return sessionToken.TimeCreated.In(date_utils.CurrentLocation()).After(ageDurationAgo)
+ })
+ return validTokens
+}
diff --git a/bank_transactions/absa_bank_transactions.go b/bank_transactions/absa_bank_transactions.go
new file mode 100644
index 0000000000000000000000000000000000000000..1f2400bd3d0327c2b86bf1325225c4ad6dd249f9
--- /dev/null
+++ b/bank_transactions/absa_bank_transactions.go
@@ -0,0 +1,274 @@
+package bank_transactions
+
+import (
+ "crypto/tls"
+ "encoding/base64"
+ "github.com/go-resty/resty/v2"
+ "github.com/google/uuid"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/secrets_manager"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/struct_utils"
+ "io"
+ "strings"
+ "time"
+)
+
+type AbsaLoginInfo struct {
+ CertExpiry string
+ CertSerialNumber string
+ PartialNo string
+ DeviceFingerprint string
+ AbsaSecretName string
+ IsDebug bool
+}
+
+var ErrPasswordChange = errors.Error("Absa CIB requires a password change")
+
+const (
+ sleepTime = time.Second * 2
+)
+
+func GetAbsaTransactions(loginInfo AbsaLoginInfo) ([]byte, error) {
+
+ client, err := setupAbsaClient(loginInfo.AbsaSecretName, loginInfo.IsDebug)
+ if err != nil {
+ return nil, err
+ }
+
+ defer logoff(client)
+
+ // Get Initial session ID
+ err = login(client, loginInfo)
+ if err != nil {
+ return nil, err
+ }
+
+ // Navigate to statement download
+ err = startStatementDownloadProcess(client)
+ if err != nil {
+ return nil, err
+ }
+
+ // Request statement
+ dateString := time.Now().Format("20060102")
+ _, err = statementDownloadProcessCall(client, map[string]string{
+ "page": "enquiry-statement",
+ "groupselect": "",
+ "linkallterm": "false",
+ "buttonClicked": "proceed",
+ "SelType": "D",
+ "FAInd": "false",
+ "systemDate": dateString,
+ "account_options": "all",
+ "fromaccountgroup": "blank",
+ "fromaccount": "blank",
+ "fromaccountdisplay1": "blank",
+ "showall_option": "showall",
+ "format_options": "file",
+ "file_format_options": "csv",
+ "statement_options": "last",
+ "days": "7",
+ "startdate": dateString,
+ "enddate": dateString,
+ "startnum": "",
+ "endnum": "",
+ "ErrorText": "",
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // Wait for statement to process
+ for {
+ responseBytes, err := statementDownloadProcessCall(client, map[string]string{
+ "page": "enquiry-statement-status",
+ "buttonClicked": "proceed",
+ "ErrorText": "",
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ if strings.Contains(string(responseBytes), "The statement file is available for download") {
+ break
+ }
+ time.Sleep(sleepTime)
+ }
+
+ // Download file
+ statementBytes, err := statementDownloadProcessCall(client, map[string]string{
+ "page": "enquiry-statement-file",
+ "buttonClicked": "saveas",
+ "filename": "25591016.csv",
+ "ErrorText": "",
+ })
+ if err != nil {
+ return nil, err
+ }
+ return statementBytes, nil
+}
+
+func setupAbsaClient(secretName string, isDebug bool) (*resty.Client, error) {
+ // Get cert data
+ credentials, err := getAbsaSecret(secretName, isDebug)
+ if err != nil {
+ return nil, err
+ }
+ certDataString := credentials["cert"]
+ certData, _ := base64.StdEncoding.DecodeString(certDataString)
+
+ privateKeyDataString := credentials["private_key"]
+ privateKeyData, _ := base64.StdEncoding.DecodeString(privateKeyDataString)
+
+ cert, err := tls.X509KeyPair(certData, privateKeyData)
+ if err != nil {
+ return nil, err
+ }
+
+ client := resty.New().
+ SetCertificates(cert)
+ return client, nil
+}
+
+func login(client *resty.Client, loginInfo AbsaLoginInfo) error {
+
+ // Get password
+ credentials, err := getAbsaSecret(loginInfo.AbsaSecretName, loginInfo.IsDebug)
+ if err != nil {
+ return err
+ }
+
+ username := credentials["username"]
+ password := credentials["password"]
+
+ // Get Initial session ID
+ response, err := client.R().
+ SetDoNotParseResponse(true).
+ Get("https://bionline.absa.co.za/businessintegrator/login_new.jsp")
+ if err != nil {
+ return err
+ }
+ time.Sleep(sleepTime)
+
+ // Login
+ universalTracker := uuid.New().String() + "-" + uuid.New().String()
+ response, err = client.R().
+ SetDoNotParseResponse(true).
+ SetFormData(map[string]string{
+ "lastpage": "login",
+ "buttonClicked": "proceed",
+ "hasCert": "Y",
+ "certExpired": "false",
+ "certExpiry": loginInfo.CertExpiry,
+ "displayIgnoreButton": "true",
+ "partialNo": loginInfo.PartialNo,
+ "universalTracker": universalTracker,
+ "deviceFingerprint": loginInfo.DeviceFingerprint,
+ "certSerialNo": loginInfo.CertSerialNumber,
+ "UserID": username,
+ "password_one": password,
+ }).
+ Post("https://bionline.absa.co.za/businessintegrator/SignOn")
+ if err != nil {
+ return err
+ }
+
+ responseBytes, err := io.ReadAll(response.RawResponse.Body)
+ if strings.Contains(string(responseBytes), "User already signed on") {
+ // Log out
+ response, err = client.R().
+ SetDoNotParseResponse(true).
+ SetFormData(map[string]string{
+ "buttonClicked": "proceed",
+ "lastpage": "signedon_to_signoff",
+ "ErrorText": "",
+ }).
+ Post("https://bionline.absa.co.za/businessintegrator/SignOn")
+ if err != nil {
+ return err
+ }
+
+ // Login again
+ err := login(client, loginInfo)
+ if err != nil {
+ return err
+ }
+ }
+
+ if strings.Contains(string(responseBytes), "New Password") {
+ return ErrPasswordChange
+ }
+
+ return nil
+}
+
+func startStatementDownloadProcess(client *resty.Client) error {
+ // Navigate to statement download
+ response, err := client.R().
+ SetDoNotParseResponse(true).
+ SetQueryParam("dojo.preventCache", string_utils.Int64ToString(time.Now().UnixMilli())).
+ SetHeader("Referer", "https://bionline.absa.co.za/businessintegrator/enquiries/menu.jsp").
+ SetHeader("Content-Type", "application/x-www-form-urlencoded").
+ Get("https://bionline.absa.co.za/businessintegrator/EnquireBankStatement")
+ if err != nil {
+ return err
+ }
+
+ responseBytes, _ := io.ReadAll(response.RawResponse.Body)
+
+ // Ensure we are on the "Statement Enquiry" screen
+ if !strings.Contains(string(responseBytes), "Statement Enquiry") {
+ return errors.New("Could not load 'Statement Enquiry' screen")
+ }
+ time.Sleep(sleepTime)
+ return nil
+}
+
+func statementDownloadProcessCall(client *resty.Client, formData map[string]string) ([]byte, error) {
+ response, err := client.R().
+ SetDoNotParseResponse(true).
+ SetFormData(formData).
+ Post("https://bionline.absa.co.za/businessintegrator/EnquireBankStatement")
+ if err != nil {
+ return nil, err
+ }
+
+ responseBytes, err := io.ReadAll(response.RawResponse.Body)
+ time.Sleep(sleepTime)
+ return responseBytes, nil
+}
+
+func logoff(client *resty.Client) error {
+ // Logoff
+ _, err := client.R().
+ SetDoNotParseResponse(true).
+ Get("https://bionline.absa.co.za/businessintegrator/Logoff")
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func getAbsaSecret(secretName string, isDebug bool) (map[string]string, error) {
+ // This secret is manually created on Secrets Manager
+
+ /*
+ Extract the key and cert from the p12 certificate
+ ```
+ openssl pkcs12 -in Absa.p12 -nocerts -out userkey.pem -nodes
+ openssl pkcs12 -in Absa.p12 -clcerts -nokeys -out usercert.pem
+ ```
+
+ Base64 encode the key and cert and save it in secrets manager
+ */
+
+ secretJson, _ := secrets_manager.GetSecret(secretName, isDebug)
+ var credentials map[string]string
+ err := struct_utils.UnmarshalJSON([]byte(secretJson), &credentials)
+ if err != nil {
+ return nil, err
+ }
+
+ return credentials, nil
+}
diff --git a/cognito/cognito.go b/cognito/cognito.go
index d8300aee61cb523bc6fd133bda6fc39e9d31dfc2..ac79da6f825d272f7e2756c675a1ede4e2eeafae 100644
--- a/cognito/cognito.go
+++ b/cognito/cognito.go
@@ -2,6 +2,8 @@ package cognito
import (
"fmt"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/utils"
"math/rand"
"strings"
@@ -90,6 +92,63 @@ func SetUserPassword(pool string, username string, password string) (*cognitoide
return output, err
}
+func confirmForgotPassword(appClientID string, username string, password string, confirmationCode string) (*cognitoidentityprovider.ConfirmForgotPasswordOutput, error) {
+ input := cognitoidentityprovider.ConfirmForgotPasswordInput{
+ ClientId: &appClientID,
+ ConfirmationCode: &confirmationCode,
+ Password: &password,
+ Username: &username,
+ }
+ output, err := CognitoService.ConfirmForgotPassword(&input)
+ logs.Info("output", output)
+ return output, err
+}
+
+func confirmPasswordReset(appClientID string, username string, password string, initiateAuthOutput *cognitoidentityprovider.InitiateAuthOutput) (*cognitoidentityprovider.RespondToAuthChallengeOutput, error) {
+ // Respond to the Auth challenge to change the user's password
+ authChallengeParameters := map[string]*string{
+ "USERNAME": utils.ValueToPointer(username),
+ "NEW_PASSWORD": utils.ValueToPointer(password),
+ }
+ respondToAuthChallengeInput := cognitoidentityprovider.RespondToAuthChallengeInput{
+ ChallengeName: initiateAuthOutput.ChallengeName,
+ ChallengeResponses: authChallengeParameters,
+ ClientId: &appClientID,
+ Session: initiateAuthOutput.Session,
+ }
+ output, err := CognitoService.RespondToAuthChallenge(&respondToAuthChallengeInput)
+ logs.Info("output", output)
+ return output, err
+}
+
+// ConfirmPasswordReset initiates a Cognito auth for the user, and based on the output either updates the user's password,
+// or performs a forgot password confirmation.
+func ConfirmPasswordReset(appClientID string, username string, password string, confirmationCode string) (interface{}, error) {
+ // Initiate an auth for the user to see if a password reset or
+ authParameters := map[string]*string{
+ "USERNAME": utils.ValueToPointer(username),
+ "PASSWORD": utils.ValueToPointer(confirmationCode),
+ }
+ initiateAuthInput := cognitoidentityprovider.InitiateAuthInput{
+ AuthFlow: utils.ValueToPointer(cognitoidentityprovider.ExplicitAuthFlowsTypeUserPasswordAuth),
+ AuthParameters: authParameters,
+ ClientId: &appClientID,
+ }
+ res, err := CognitoService.InitiateAuth(&initiateAuthInput)
+ if err != nil {
+ if errors.AWSErrorExceptionCode(err) == cognitoidentityprovider.ErrCodePasswordResetRequiredException {
+ // Not a user verification - perform forgot password confirmation
+ return confirmForgotPassword(appClientID, username, password, confirmationCode)
+ }
+ return nil, err
+ }
+ if utils.PointerToValue(res.ChallengeName) == cognitoidentityprovider.ChallengeNameTypeNewPasswordRequired {
+ return confirmPasswordReset(appClientID, username, password, res)
+ }
+
+ return nil, errors.New("User state not correct for confirmation. Please contact support.")
+}
+
// FOR API LOGS
func DetermineAuthType(identity events.APIGatewayRequestIdentity) *string {
diff --git a/date_utils/date_utils.go b/date_utils/date_utils.go
index e456d7977992d96d2a3ac13c2525f9861e6afb61..d60b7c8255b1bc500c9585f41a32a93e0259839e 100644
--- a/date_utils/date_utils.go
+++ b/date_utils/date_utils.go
@@ -8,6 +8,8 @@ import (
const TimeZoneString = "Africa/Johannesburg"
+var currentLocation *time.Location
+
func DateLayoutYearMonthDayTimeT() string {
layout := "2006-01-02T15:04:05"
return layout
@@ -81,8 +83,10 @@ func DateDBFormattedStringDateOnly(date time.Time) string {
}
func CurrentLocation() *time.Location {
- loc, _ := time.LoadLocation(TimeZoneString)
- return loc
+ if currentLocation == nil {
+ currentLocation, _ = time.LoadLocation(TimeZoneString)
+ }
+ return currentLocation
}
func DateLocal(date *time.Time) {
diff --git a/encryption/encryption.go b/encryption/encryption.go
index 909d48cc9936341be6191e7165422cc25ae7a029..9d06d057de8806812f23988d817fd48ab766db24 100644
--- a/encryption/encryption.go
+++ b/encryption/encryption.go
@@ -1,6 +1,7 @@
package encryption
import (
+ "bytes"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
@@ -8,6 +9,8 @@ import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
+ "encoding/hex"
+ "encoding/json"
"fmt"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
"io"
@@ -20,48 +23,138 @@ func Hash(input string, key string) string {
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
+// GenerateHashFromObject using HMAC with SHA-256
+func GenerateHashFromObject(object any, secret string) (string, error) {
+ // Base64 Encode body
+ var buf bytes.Buffer
+ encoder := base64.NewEncoder(base64.StdEncoding, &buf)
+ defer encoder.Close()
+
+ err := json.NewEncoder(encoder).Encode(object)
+ if err != nil {
+ return "", err
+ }
+ encodedBody := buf.String()
+
+ // Sign encoded body with secret
+ hash := hmac.New(sha256.New, []byte(secret))
+ hash.Write([]byte(encodedBody))
+ hashedBody := hex.EncodeToString(hash.Sum(nil))
+ return hashedBody, nil
+}
+
func Md5HashString(bytesToHash []byte) string {
hash := md5.Sum(bytesToHash)
hashString := fmt.Sprintf("%X", hash)
return hashString
}
+func EncryptStruct(object any, key string) (string, error) {
+ if len(key) != 32 {
+ return "", errors.New("key should be 32 bytes")
+ }
+
+ block, err := aes.NewCipher([]byte(key))
+ if err != nil {
+ return "", err
+ }
+
+ aesGcm, err := cipher.NewGCM(block)
+ if err != nil {
+ return "", err
+ }
+
+ nonce := make([]byte, aesGcm.NonceSize())
+ if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+ return "", err
+ }
+
+ jsonValue, err := json.Marshal(object)
+ if err != nil {
+ return "", err
+ }
+
+ encryptedValue := string(aesGcm.Seal(nonce, nonce, jsonValue, nil))
+ return base64.StdEncoding.EncodeToString([]byte(encryptedValue)), nil
+}
+
+func DecryptStruct(encryptedStruct string, key string, object any) error {
+ if len(key) != 32 {
+ return errors.New("key should be 32 bytes")
+ }
+
+ decodedStruct, _ := base64.StdEncoding.DecodeString(encryptedStruct)
+ decodedStructString := string(decodedStruct)
+
+ block, err := aes.NewCipher([]byte(key))
+ if err != nil {
+ return err
+ }
+
+ aesGcm, err := cipher.NewGCM(block)
+ if err != nil {
+ return err
+ }
+
+ nonceSize := aesGcm.NonceSize()
+ if len(decodedStructString) < nonceSize {
+ return errors.New("ciphertext too short")
+ }
+
+ nonce, ciphertext := decodedStructString[:nonceSize], decodedStructString[nonceSize:]
+ value, err := aesGcm.Open(nil, []byte(nonce), []byte(ciphertext), nil)
+
+ err = json.Unmarshal(value, object)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
func Encrypt(plaintext string, key string) (string, error) {
- c, err := aes.NewCipher([]byte(key))
+ if len(key) != 32 {
+ return "", errors.New("key should be 32 bytes")
+ }
+
+ block, err := aes.NewCipher([]byte(key))
if err != nil {
return "", err
}
- gcm, err := cipher.NewGCM(c)
+ aesGcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
- nonce := make([]byte, gcm.NonceSize())
+ nonce := make([]byte, aesGcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return "", err
}
- return string(gcm.Seal(nonce, nonce, []byte(plaintext), nil)), nil
+ return string(aesGcm.Seal(nonce, nonce, []byte(plaintext), nil)), nil
}
func Decrypt(ciphertext string, key string) (string, error) {
- c, err := aes.NewCipher([]byte(key))
+ if len(key) != 32 {
+ return "", errors.New("key should be 32 bytes")
+ }
+
+ block, err := aes.NewCipher([]byte(key))
if err != nil {
return "", err
}
- gcm, err := cipher.NewGCM(c)
+ aesGcm, err := cipher.NewGCM(block)
if err != nil {
return "", err
}
- nonceSize := gcm.NonceSize()
+ nonceSize := aesGcm.NonceSize()
if len(ciphertext) < nonceSize {
return "", errors.New("ciphertext too short")
}
nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
- value, err := gcm.Open(nil, []byte(nonce), []byte(ciphertext), nil)
+ value, err := aesGcm.Open(nil, []byte(nonce), []byte(ciphertext), nil)
return string(value), err
}
diff --git a/encryption/encryption_keys.go b/encryption/encryption_keys.go
new file mode 100644
index 0000000000000000000000000000000000000000..5d94a0f8ebf5d3aec3ea2cbca5b3de67ec25935d
--- /dev/null
+++ b/encryption/encryption_keys.go
@@ -0,0 +1,56 @@
+package encryption
+
+import (
+ "encoding/json"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/secrets_manager"
+)
+
+type EncryptionKeysSecret struct {
+ EncryptionKeysValue string `json:"EncryptionKeys"`
+}
+
+type EncryptionKeys struct {
+ FirebaseEncryptionKey string `json:"firebase_encryption_key"`
+ JWTEncryptionKey string `json:"jwt_encryption_key"`
+ BobAPIAuthEncryptionKey string `json:"bob_api_auth_encryption_key"`
+ GenericEncryptionKey string `json:"generic_encryption_key"`
+}
+
+func GetEncryptionKeys(secretID string, isDebug bool) (EncryptionKeys, error) {
+ encryptionKeysSecretString, _ := secrets_manager.GetSecret(secretID, isDebug)
+
+ var encryptionKeys EncryptionKeys
+ var encryptionKeysSecret EncryptionKeysSecret
+ err := json.Unmarshal([]byte(encryptionKeysSecretString), &encryptionKeysSecret)
+ if err == nil {
+ err = json.Unmarshal([]byte(encryptionKeysSecret.EncryptionKeysValue), &encryptionKeys)
+ }
+
+ return encryptionKeys, err
+}
+
+func GetJWTEncryptionKey(secretID string, isDebug bool) (string, error) {
+ encryptionKeys, err := GetEncryptionKeys(secretID, isDebug)
+ if err != nil {
+ logs.ErrorWithMsg("Could not get encryption keys from secret manager", err)
+ return "", errors.Error("failed to get encryption keys for login")
+ }
+ return encryptionKeys.JWTEncryptionKey, nil
+}
+
+func GetFirebaseCredentialsEncryptionKey(secretID string, isDebug bool) (string, error) {
+ encryptionKeys, err := GetEncryptionKeys(secretID, isDebug)
+ return encryptionKeys.FirebaseEncryptionKey, err
+}
+
+func GetBobAPIAuthEncryptionKey(secretID string, isDebug bool) (string, error) {
+ encryptionKeys, err := GetEncryptionKeys(secretID, isDebug)
+ return encryptionKeys.BobAPIAuthEncryptionKey, err
+}
+
+func GetGenericEncryptionKey(secretID string, isDebug bool) (string, error) {
+ encryptionKeys, err := GetEncryptionKeys(secretID, isDebug)
+ return encryptionKeys.GenericEncryptionKey, err
+}
diff --git a/errors/errors.go b/errors/errors.go
index d5a5556e4581ee8c7d3ecb6f249fd75b43f03997..bd6c2afebeb32fa6e72e7c8ef74685757ca87d92 100644
--- a/errors/errors.go
+++ b/errors/errors.go
@@ -2,6 +2,7 @@ package errors
import (
"fmt"
+ "github.com/aws/aws-sdk-go/aws/awserr"
pkg_errors "github.com/pkg/errors"
)
@@ -117,6 +118,28 @@ func HTTPWithError(code int, err error) error {
return wrappedErr
}
+func AWSErrorExceptionCode(err error) string {
+ if err == nil {
+ return ""
+ }
+
+ if awsError, ok := err.(awserr.Error); ok {
+ return awsError.Code()
+ }
+ return ""
+}
+
+func AWSErrorWithoutExceptionCode(err error) error {
+ if err == nil {
+ return nil
+ }
+
+ if awsError, ok := err.(awserr.Error); ok {
+ return Error(awsError.Message())
+ }
+ return err
+}
+
type Description struct {
Message string `json:"error"`
Source *CallerInfo `json:"source,omitempty"`
diff --git a/go.mod b/go.mod
index 5a4c62654be9c4150966208b709b51206a78232d..aca98a385b2c0173e38560e63979b2d3dea8aa93 100644
--- a/go.mod
+++ b/go.mod
@@ -6,19 +6,22 @@ require (
github.com/MindscapeHQ/raygun4go v1.1.1
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/aws/aws-lambda-go v1.26.0
- github.com/aws/aws-sdk-go v1.43.2
+ github.com/aws/aws-sdk-go v1.44.180
github.com/aws/aws-secretsmanager-caching-go v1.1.0
github.com/go-pg/pg/v10 v10.10.6
github.com/go-redis/redis/v8 v8.11.4
github.com/go-redis/redis_rate/v9 v9.1.2
+ github.com/go-resty/resty/v2 v2.7.0
+ github.com/golang-jwt/jwt/v4 v4.4.3
github.com/google/uuid v1.3.0
- github.com/opensearch-project/opensearch-go v1.1.0
+ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
+ github.com/opensearch-project/opensearch-go/v2 v2.2.0
github.com/pkg/errors v0.9.1
github.com/r3labs/diff/v2 v2.14.2
github.com/sirupsen/logrus v1.8.1
github.com/thoas/go-funk v0.9.1
- gitlab.com/uafrica/go-utils v1.49.0
- golang.org/x/text v0.3.7
+ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+ golang.org/x/text v0.4.0
)
require (
@@ -27,21 +30,18 @@ require (
github.com/go-errors/errors v1.4.1 // indirect
github.com/go-pg/zerochecker v0.2.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
- github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
- github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/pborman/uuid v1.2.1 // indirect
- github.com/smartystreets/assertions v1.2.0 // indirect
+ github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/vmihailenco/bufpool v0.1.11 // indirect
github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
- golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
- golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
- golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect
+ golang.org/x/net v0.1.0 // indirect
+ golang.org/x/sys v0.1.0 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.26.0 // indirect
mellium.im/sasl v0.2.1 // indirect
diff --git a/go.sum b/go.sum
index ba9a7ff8cf137b2fb589ce03e67a0e562766a7a1..f9f519cc4cde8de1478fd87b1bdff77ce56d799d 100644
--- a/go.sum
+++ b/go.sum
@@ -7,11 +7,22 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoU
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go v1.19.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
-github.com/aws/aws-sdk-go v1.43.2 h1:T6LuKCNu8CYXXDn3xJoldh8FbdvuVH7C9aSuLNrlht0=
-github.com/aws/aws-sdk-go v1.43.2/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/aws/aws-sdk-go v1.44.180 h1:VLZuAHI9fa/3WME5JjpVjcPCNfpGHVMiHx8sLHWhMgI=
+github.com/aws/aws-sdk-go v1.44.180/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2/config v1.18.8/go.mod h1:5XCmmyutmzzgkpk/6NYTjeWb6lgo9N170m1j6pQkIBs=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.8/go.mod h1:lVa4OHbvgjVot4gmh1uouF1ubgexSCN92P6CJQpT0t8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k=
+github.com/aws/aws-sdk-go-v2/service/sso v1.12.0/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.0/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.18.0/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I=
github.com/aws/aws-secretsmanager-caching-go v1.1.0 h1:vcV94XGJ9KouXKYBTMqgrBw96Tae8JKLmoUZ5SbaXNo=
github.com/aws/aws-secretsmanager-caching-go v1.1.0/go.mod h1:wahQpJP1dZKMqjGFAjGCqilHkTlN0zReGWocPLbXmxg=
+github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -38,7 +49,11 @@ github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F4
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ=
github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ=
+github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY=
+github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
+github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -61,8 +76,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -83,6 +99,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -98,8 +116,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
-github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M=
-github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo=
+github.com/opensearch-project/opensearch-go/v2 v2.2.0 h1:6RicCBiqboSVtLMjSiKgVQIsND4I3sxELg9uwWe/TKM=
+github.com/opensearch-project/opensearch-go/v2 v2.2.0/go.mod h1:R8NTTQMmfSRsmZdfEn2o9ZSuSXn0WTHPYhzgl7LCFLY=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -118,13 +136,19 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
@@ -141,8 +165,7 @@ github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-gitlab.com/uafrica/go-utils v1.49.0 h1:mmMHmOHQ57wKaD3zmOujlwjsCNHA52mAlrmt3MlKKAE=
-gitlab.com/uafrica/go-utils v1.49.0/go.mod h1:jSqj/FKsBEceSuxg2oKx9fsHxyWHvxJe7bJNAtWRGBw=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -154,6 +177,7 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -167,13 +191,16 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -189,26 +216,33 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 h1:c20P3CcPbopVp2f7099WLOqSNKURf30Z0uq66HpijZY=
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -246,8 +280,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w=
diff --git a/handler_utils/debug.go b/handler_utils/debug.go
new file mode 100644
index 0000000000000000000000000000000000000000..06817f60158d452ff4db373db3003365b913a2a7
--- /dev/null
+++ b/handler_utils/debug.go
@@ -0,0 +1,122 @@
+package handler_utils
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/aws/aws-lambda-go/events"
+ "github.com/aws/aws-lambda-go/lambda"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/utils"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+)
+
+// ========================================================================
+// HTTP functions
+// ========================================================================
+
+func ServeHTTPFunctions(ctx context.Context, lambdaHandler lambda.Handler, w http.ResponseWriter, req *http.Request) {
+
+ // Read body
+ buf := new(bytes.Buffer)
+ _, err := buf.ReadFrom(req.Body)
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+ req.Body.Close()
+ body := buf.String()
+
+ // Read Query
+ queryValues := req.URL.Query()
+ query := map[string]string{}
+ for key, values := range queryValues {
+ query[key] = values[0]
+ }
+
+ headers := map[string]string{}
+ for key, values := range req.Header {
+ if len(values) > 0 {
+ headers[key] = values[0]
+ }
+ }
+
+ // Call lambda function
+ request := events.APIGatewayProxyRequest{
+ Path: req.URL.Path,
+ Resource: req.URL.Path,
+ HTTPMethod: req.Method,
+ QueryStringParameters: query,
+ Body: body,
+ Headers: headers,
+ }
+
+ jsonRequest, _ := json.Marshal(request)
+ responseBytes, err := lambdaHandler.Invoke(ctx, jsonRequest)
+ if err != nil {
+ panic(err)
+ }
+
+ // Parse response
+ response := map[string]any{}
+ err = json.Unmarshal(responseBytes, &response)
+ if err != nil {
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(int(response["statusCode"].(float64)))
+ _, _ = io.WriteString(w, response["body"].(string))
+}
+
+// ========================================================================
+// SQS Functions
+// ========================================================================
+
+func ServeSQSFunctions(ctx context.Context, lambdaHandler lambda.Handler, w http.ResponseWriter, req *http.Request) {
+
+ // Read body
+ buf := new(bytes.Buffer)
+ _, err := buf.ReadFrom(req.Body)
+ if err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+ req.Body.Close()
+ body := buf.String()
+
+ sqsType := req.URL.Path
+ sqsType = strings.ReplaceAll(sqsType, "/sqs/", "")
+
+ // Call lambda function
+ sqsRequest, _ := json.Marshal(events.SQSEvent{
+ Records: []events.SQSMessage{
+ {
+ Body: body,
+ MessageAttributes: map[string]events.SQSMessageAttribute{
+ "type": {
+ StringValue: utils.ValueToPointer(sqsType),
+ },
+ },
+ },
+ },
+ })
+
+ responseBytes, err := lambdaHandler.Invoke(ctx, sqsRequest)
+ if err != nil {
+ panic(err)
+ }
+
+ // Parse response
+ response := map[string]any{}
+ err = json.Unmarshal(responseBytes, &response)
+ if err != nil {
+ _, _ = io.WriteString(w, err.Error())
+ return
+ }
+
+ _, _ = io.WriteString(w, "done")
+}
diff --git a/handler_utils/request.go b/handler_utils/request.go
index f3e4260698d68fa0857d607ef76d08870ed40ff8..5633f913580ac8f235a8dd63744a68386ef6411b 100644
--- a/handler_utils/request.go
+++ b/handler_utils/request.go
@@ -1,7 +1,16 @@
package handler_utils
import (
+ "bytes"
"context"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+ "github.com/go-resty/resty/v2"
+ "io"
+ "net/http"
+ "os"
+ "strings"
+ "time"
"github.com/aws/aws-lambda-go/lambdacontext"
)
@@ -30,3 +39,39 @@ func AddRequestIDToHeaders(requestID *string, headers map[string]string, request
headers[requestIDHeaderKey] = *requestID
}
}
+
+func SignAWSRestyClient(client *resty.Client, accessKeyID, secretAccessKey string, bodyBytes []byte) error {
+ var bodySeeker io.ReadSeeker = bytes.NewReader(bodyBytes)
+
+ // Add a hook to sign the request before sending
+ client.SetPreRequestHook(func(_ *resty.Client, r *http.Request) error {
+ err := SignAWSHttpRequest(r, accessKeyID, secretAccessKey, bodySeeker)
+ if err != nil {
+ return err
+ }
+ return nil
+ })
+
+ return nil
+}
+
+// SignAWSRequest wraps and executes http.NewRequest and adds a sig version 4 signature for AWS API Gateway
+func SignAWSHttpRequest(request *http.Request, accessKeyID, secretAccessKey string, bodySeeker io.ReadSeeker) error {
+ // Use AWS SDK to sign the request for API gateway, i.e. execute-api, and the current region
+ _, err := v4.NewSigner(credentials.NewStaticCredentials(accessKeyID, secretAccessKey, "")).
+ Sign(request, bodySeeker, "execute-api", os.Getenv("AWS_REGION"), time.Now())
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func FindHeaderValue(headers map[string]string, key string) string {
+ for k, v := range headers {
+ if strings.ToLower(k) == strings.ToLower(key) {
+ return v
+ }
+ }
+ return ""
+}
diff --git a/logs/logs.go b/logs/logs.go
index 75f7d79df494b921811b6b3bb0c6359fd44d7c45..f1b4c59aeaa9b30030770817e48c4a6d30c22bc1 100644
--- a/logs/logs.go
+++ b/logs/logs.go
@@ -3,9 +3,13 @@ package logs
import (
"errors"
"fmt"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/utils"
"net/http"
"net/url"
"os"
+ "reflect"
+ "regexp"
"runtime"
"strings"
@@ -26,6 +30,59 @@ var raygunClient *raygun4go.Client
// TODO
// Sensitive word filtering
+// Password filtering
+var passwordRegex = regexp.MustCompile(`(?i:\\?"password\\?"\s*:\s*\\?"(.*)\\?",).*`)
+
+func SanitiseLogs(logString string) string {
+ logString = MaskPasswordsInJsonString(logString)
+
+ return logString
+}
+
+// MaskPasswordsInJsonString takes a string and, if it is a JSON string, sanitises all the password. In order for the
+// regex to work correctly we need to prettify the JSON, so the function always returns a formatted JSON string.
+func MaskPasswordsInJsonString(jsonString string) string {
+ var isValidJsonString bool
+ isValidJsonString, jsonString = string_utils.PrettyJSON(jsonString)
+ if !isValidJsonString {
+ return jsonString
+ }
+
+ if passwordRegex.MatchString(jsonString) {
+ result := passwordRegex.FindAllStringSubmatch(jsonString, -1)
+ for _, match := range result {
+ if len(match) > 1 {
+ jsonString = strings.ReplaceAll(jsonString, match[1], "***")
+ }
+ }
+ }
+
+ return jsonString
+}
+
+func SanitiseFields(fields map[string]interface{}) map[string]interface{} {
+ sanitisedFields := make(map[string]interface{})
+
+ // Check if each field is a string or string pointer, and sanitize them if they are
+ for key, field := range fields {
+ value := reflect.ValueOf(field)
+ if value.Kind() == reflect.Ptr && value.IsValid() {
+ pointerValue := value.Elem()
+ if pointerValue.Kind() == reflect.String {
+ sanitisedString := SanitiseLogs(pointerValue.String())
+ sanitisedFields[key] = &sanitisedString
+ }
+ } else if value.Kind() == reflect.String {
+ sanitisedFields[key] = SanitiseLogs(value.String())
+ } else {
+ // Don't sanitise fields that
+ sanitisedFields[key] = field
+ }
+ }
+
+ return sanitisedFields
+}
+
func InitLogs(requestID *string, isDebugBuild bool, buildVersion string, request *events.APIGatewayProxyRequest, client *raygun4go.Client) {
currentRequestID = requestID
apiRequest = request
@@ -114,16 +171,22 @@ func getLogger() *log.Entry {
}
func InfoWithFields(fields map[string]interface{}, message interface{}) {
- getLogger().WithFields(fields).Info(message)
+ if reflect.TypeOf(message).Kind() == reflect.String {
+ message = SanitiseLogs(message.(string))
+ }
+ sanitisedFields := SanitiseFields(fields)
+ getLogger().WithFields(sanitisedFields).Info(message)
}
func Info(format string, a ...interface{}) {
- getLogger().Info(fmt.Sprintf(format, a...))
+ message := SanitiseLogs(fmt.Sprintf(format, a...))
+ getLogger().Info(message)
}
func ErrorWithFields(fields map[string]interface{}, err error) {
- sendRaygunError(fields, err)
- getLogger().WithFields(fields).Error(err)
+ sanitisedFields := SanitiseFields(fields)
+ sendRaygunError(sanitisedFields, err)
+ getLogger().WithFields(sanitisedFields).Error(err)
}
func ErrorWithMsg(message string, err error) {
@@ -140,11 +203,13 @@ func ErrorMsg(message string) {
}
func Warn(format string, a ...interface{}) {
- getLogger().Warn(fmt.Sprintf(format, a...))
+ message := SanitiseLogs(fmt.Sprintf(format, a...))
+ getLogger().Warn(message)
}
func WarnWithFields(fields map[string]interface{}, err error) {
- getLogger().WithFields(fields).Warn(err)
+ sanitisedFields := SanitiseFields(fields)
+ getLogger().WithFields(sanitisedFields).Warn(err)
}
func SQLDebugInfo(sql string) {
@@ -213,7 +278,8 @@ func sendRaygunError(fields map[string]interface{}, errToSend error) {
}
fields["env"] = env
- raygunClient.CustomData(fields)
+ sanitisedFields := SanitiseFields(fields)
+ raygunClient.CustomData(sanitisedFields)
raygunClient.Request(fakeHttpRequest())
if errToSend == nil {
@@ -230,6 +296,15 @@ func fakeHttpRequest() *http.Request {
return nil
}
+ // Mask authorization header for raygun logs
+ headers := utils.DeepCopy(apiRequest.MultiValueHeaders).(map[string][]string)
+ if len(headers["authorization"]) != 0 {
+ headers["authorization"] = []string{"***"}
+ }
+ if len(headers["Authorization"]) != 0 {
+ headers["Authorization"] = []string{"***"}
+ }
+
requestURL := url.URL{
Path: apiRequest.Path,
Host: apiRequest.Headers["Host"],
@@ -237,7 +312,7 @@ func fakeHttpRequest() *http.Request {
request := http.Request{
Method: apiRequest.HTTPMethod,
URL: &requestURL,
- Header: apiRequest.MultiValueHeaders,
+ Header: headers,
}
return &request
}
diff --git a/map_utils/map_utils.go b/map_utils/map_utils.go
new file mode 100644
index 0000000000000000000000000000000000000000..5a41270d8bb5059d5572f5de954ad11190e28eef
--- /dev/null
+++ b/map_utils/map_utils.go
@@ -0,0 +1,53 @@
+package map_utils
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+// MapStringInterfaceToMapStringString converts a generic value typed map to a map with string values
+func MapStringInterfaceToMapStringString(inputMap map[string]interface{}) map[string]string {
+ query := make(map[string]string)
+ for mapKey, mapVal := range inputMap {
+ // Check if mapVal is a slice or a single value
+ switch mapValTyped := mapVal.(type) {
+ case []interface{}:
+ // Slice - convert each element individually
+ var mapValString []string
+
+ // Loop through each element in the slice and check the type
+ for _, sliceElem := range mapValTyped {
+ switch sliceElemTyped := sliceElem.(type) {
+ case string:
+ // Enclose strings in escaped quotations
+ mapValString = append(mapValString, fmt.Sprintf("\"%v\"", sliceElemTyped))
+ case float64:
+ // Use FormatFloat for least amount of precision.
+ mapValString = append(mapValString, strconv.FormatFloat(sliceElemTyped, 'f', -1, 64))
+ default:
+ // Convert to string
+ mapValString = append(mapValString, fmt.Sprintf("%v", sliceElemTyped))
+ }
+ }
+ // Join as a comma seperated array
+ query[mapKey] = "[" + strings.Join(mapValString, ",") + "]"
+ default:
+ // Single value - convert to string
+ query[mapKey] = fmt.Sprintf("%v", mapVal)
+ }
+ }
+
+ return query
+}
+
+// MergeMaps If there are similar properties in the maps, the last one will be used as the value
+func MergeMaps(maps ...map[string]string) map[string]string {
+ ret := map[string]string{}
+ for _, mapV := range maps {
+ for k, v := range mapV {
+ ret[k] = v
+ }
+ }
+ return ret
+}
diff --git a/number_utils/number_utils.go b/number_utils/number_utils.go
index d183fe9fbcaef6b730943dcd276cf82039a486b2..9c4221b6bcec705627f3f27889a4f50b6c5ad29f 100644
--- a/number_utils/number_utils.go
+++ b/number_utils/number_utils.go
@@ -1,6 +1,9 @@
package number_utils
-import "math"
+import (
+ "math"
+ "strconv"
+)
func RoundFloat(value float64) float64 {
return math.Round(value*100) / 100 // 2 decimal places
@@ -11,16 +14,19 @@ func RoundFloatTo(value float64, to int) float64 {
return math.Round(value*toValue) / toValue
}
-func UnwrapFloat64(f *float64) float64 {
- if f == nil {
- return 0.0
- }
- return *f
+func StringToInt(stringValue string) (int, error) {
+ return strconv.Atoi(stringValue)
+}
+
+func StringToInt64(stringValue string) (int64, error) {
+ number, err := strconv.ParseInt(stringValue, 10, 64)
+ return number, err
}
-func UnwrapInt64(i *int64) int64 {
- if i == nil {
- return 0
+func StringToFloat64(stringValue string) (float64, error) {
+ number, err := strconv.ParseFloat(stringValue, 64)
+ if err != nil {
+ return 0, err
}
- return *i
+ return number, nil
}
diff --git a/oauth/oauth.go b/oauth/oauth.go
new file mode 100644
index 0000000000000000000000000000000000000000..d0c4e1d085c93e16eb26f564e073e07ddd50251c
--- /dev/null
+++ b/oauth/oauth.go
@@ -0,0 +1,79 @@
+package oauth
+
+import (
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/google/uuid"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/encryption"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type Oauth1 struct {
+ ConsumerKey string
+ ConsumerSecret string
+ AccessToken string
+ AccessSecret string
+}
+
+func (auth Oauth1) GenerateAuthorizationHeader(method, requestUrl string) (AuthorizationValue *string, err error) {
+
+ // Take the URL and get all its parts
+ urlParts, err := url.Parse(requestUrl)
+ if err != nil {
+ return nil, err
+ }
+
+ // Get the query parameters from the URL and make it easier to work with
+ rawParams, err := url.ParseQuery(urlParts.RawQuery)
+ if err != nil {
+ return nil, err
+ }
+ params := make(map[string]string)
+ for key, _ := range rawParams {
+ params[key] = rawParams.Get(key)
+ }
+
+ // Make a nonce
+ nonce := strings.ReplaceAll(uuid.New().String(), "-", "")
+
+ urlValues := url.Values{}
+ urlValues.Add("oauth_nonce", nonce)
+ urlValues.Add("oauth_consumer_key", auth.ConsumerKey)
+ urlValues.Add("oauth_signature_method", "HMAC-SHA256")
+ urlValues.Add("oauth_timestamp", strconv.Itoa(int(time.Now().Unix())))
+ urlValues.Add("oauth_token", auth.AccessToken)
+ urlValues.Add("oauth_version", "1.0")
+
+ for k, v := range params {
+ urlValues.Add(k, v)
+ }
+
+ // If there are any '+' encoded spaces replace them with the proper urlencoded version of a space
+ parameterString := strings.Replace(urlValues.Encode(), "+", "%20", -1)
+
+ // Build the signature
+ signatureBase := strings.ToUpper(method) + "&" + url.QueryEscape(strings.Split(requestUrl, "?")[0]) + "&" + url.QueryEscape(parameterString)
+ signingKey := url.QueryEscape(auth.ConsumerSecret) + "&" + url.QueryEscape(auth.AccessSecret)
+ signature := encryption.Hash(signatureBase, signingKey)
+
+ // Populate all the authorisation parameters
+ authParams := map[string]string{
+ "oauth_consumer_key": url.QueryEscape(urlValues.Get("oauth_consumer_key")),
+ "oauth_nonce": url.QueryEscape(urlValues.Get("oauth_nonce")),
+ "oauth_signature": url.QueryEscape(signature),
+ "oauth_signature_method": url.QueryEscape(urlValues.Get("oauth_signature_method")),
+ "oauth_timestamp": url.QueryEscape(urlValues.Get("oauth_timestamp")),
+ "oauth_token": url.QueryEscape(urlValues.Get("oauth_token")),
+ "oauth_version": url.QueryEscape(urlValues.Get("oauth_version")),
+ }
+
+ // Convert all the parameters into a comma delimited string with values defined as ""
+ var AuthorizationString string
+ for k, v := range authParams {
+ AuthorizationString += k + "=\"" + v + "\","
+ }
+
+ return aws.String("OAuth " + strings.TrimSuffix(AuthorizationString, ",")), nil
+}
diff --git a/search/README.md b/opensearch/README.md
similarity index 100%
rename from search/README.md
rename to opensearch/README.md
diff --git a/search/config.go b/opensearch/config.go
similarity index 98%
rename from search/config.go
rename to opensearch/config.go
index 4f1a112a7b1aa4b4ad621b8b2ddbf0073a22dacf..622649e1b81a97fb24c974a196f984245a7125f7 100644
--- a/search/config.go
+++ b/opensearch/config.go
@@ -1,4 +1,4 @@
-package search
+package opensearch
import (
"regexp"
diff --git a/search/dev/docker-compose.yml b/opensearch/dev/docker-compose.yml
similarity index 59%
rename from search/dev/docker-compose.yml
rename to opensearch/dev/docker-compose.yml
index ba1b3fd185900d6d1c2528f4172aa4fa008b144e..0dc8e6d6593d6b5ec4bec7ee53a05ce77293e133 100644
--- a/search/dev/docker-compose.yml
+++ b/opensearch/dev/docker-compose.yml
@@ -1,13 +1,13 @@
version: '3'
services:
- opensearch-node1:
+ opensearch-node1A:
image: opensearchproject/opensearch:latest
- container_name: opensearch-node1
+ container_name: opensearch-node1A
environment:
- cluster.name=opensearch-cluster
- - node.name=opensearch-node1
- - discovery.seed_hosts=opensearch-node1,opensearch-node2
- - cluster.initial_master_nodes=opensearch-node1,opensearch-node2
+ - node.name=opensearch-node1A
+ - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
+ - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
ulimits:
@@ -24,14 +24,14 @@ services:
- 9600:9600 # required for Performance Analyzer
networks:
- opensearch-net
- opensearch-node2:
+ opensearch-node2A:
image: opensearchproject/opensearch:latest
- container_name: opensearch-node2
+ container_name: opensearch-node2A
environment:
- cluster.name=opensearch-cluster
- - node.name=opensearch-node2
- - discovery.seed_hosts=opensearch-node1,opensearch-node2
- - cluster.initial_master_nodes=opensearch-node1,opensearch-node2
+ - node.name=opensearch-node2A
+ - discovery.seed_hosts=opensearch-node1A,opensearch-node2A
+ - cluster.initial_master_nodes=opensearch-node1A,opensearch-node2A
- bootstrap.memory_lock=true
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
ulimits:
@@ -45,30 +45,20 @@ services:
- opensearch-data2:/usr/share/opensearch/data
networks:
- opensearch-net
- opensearch-dashboards:
+ opensearch-dashboardsA:
image: opensearchproject/opensearch-dashboards:latest
- container_name: opensearch-dashboards
+ container_name: opensearch-dashboardsA
ports:
- 5601:5601
expose:
- "5601"
environment:
- OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]'
+ OPENSEARCH_HOSTS: '["https://opensearch-node1A:9200","https://opensearch-node2A:9200"]'
networks:
- opensearch-net
-
- kibana:
- container_name: opensearch-kibana
- image: docker.elastic.co/kibana/kibana:7.11.0
- depends_on:
- - opensearch-node1
- - opensearch-node2
- ports:
- - 5602:5601
- expose:
- - "5602"
- environment:
- ELASTICSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]'
+ logstash-with-plugin:
+ image: opensearchproject/logstash-oss-with-opensearch-output-plugin:latest
+ container_name: logstash-with-plugin
networks:
- opensearch-net
@@ -77,4 +67,4 @@ volumes:
opensearch-data2:
networks:
- opensearch-net:
+ opensearch-net:
\ No newline at end of file
diff --git a/search/document_store.go b/opensearch/document_store.go
similarity index 89%
rename from search/document_store.go
rename to opensearch/document_store.go
index 3e4c440d4c74116fcd106391939580f5ba80277d..542a4af4178ca2f50057007a1816eb4eccf18035 100644
--- a/search/document_store.go
+++ b/opensearch/document_store.go
@@ -1,15 +1,15 @@
-package search
+package opensearch
import (
"bytes"
"context"
"encoding/json"
+ "github.com/opensearch-project/opensearch-go/v2/opensearchapi"
"io/ioutil"
"net/http"
"reflect"
"strings"
- opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
)
@@ -27,7 +27,7 @@ type DocumentStore struct {
// NewDocumentStore purpose:
//
-// create a document store index to write e.g. orders then allow one to search them
+// create a document store index to write e.g. orders then allow one to opensearch them
//
// parameters:
//
@@ -100,7 +100,7 @@ func (ds *DocumentStore) Write(id string, data interface{}) error {
return errors.Errorf("cannot write %T into DocumentStore(%s), expecting %s", data, ds.name, ds.dataType)
}
- // get daily search index to write to, from start time
+ // get daily opensearch index to write to, from start time
indexName := ds.name // + "-" + startTime.Format("20060102")
if !ds.created {
res, err := ds.w.api.Create(
@@ -108,6 +108,9 @@ func (ds *DocumentStore) Write(id string, data interface{}) error {
indexName, // document id
strings.NewReader(string(ds.jsonSettings)))
if err != nil {
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Wrapf(err, "failed to create index(%s)", indexName)
}
switch res.StatusCode {
@@ -115,6 +118,9 @@ func (ds *DocumentStore) Write(id string, data interface{}) error {
case http.StatusCreated:
case http.StatusConflict: // 409 = already exists
default:
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Errorf("failed to create index(%s): %v %s %s", indexName, res.StatusCode, res.Status(), res.String())
}
@@ -123,6 +129,9 @@ func (ds *DocumentStore) Write(id string, data interface{}) error {
Body: strings.NewReader(string(ds.jsonMappings)),
}.Do(context.Background(), ds.w.client)
if err != nil {
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Wrapf(err, "failed to create index(%s)", indexName)
}
switch res.StatusCode {
@@ -130,14 +139,17 @@ func (ds *DocumentStore) Write(id string, data interface{}) error {
case http.StatusCreated:
case http.StatusConflict: // 409 = already exists
default:
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Errorf("failed to create index(%s): %v %s %s", indexName, res.StatusCode, res.Status(), res.String())
}
ds.created = true
}
- if res, err := ds.w.Write(indexName, id, data); err != nil {
+ if indexResponse, err := ds.w.Write(indexName, id, data); err != nil {
return err
} else {
- logs.Info("IndexResponse: %+v", res)
+ logs.Info("IndexResponse: %+v", indexResponse)
}
return nil
}
@@ -155,7 +167,7 @@ func (ds *DocumentStore) Search(query Query, limit int64) (res *SearchResponseHi
return
}
- // example search request body for free text
+ // example opensearch request body for free text
// {
// "size": 5,
// "query": {
@@ -179,7 +191,7 @@ func (ds *DocumentStore) Search(query Query, limit int64) (res *SearchResponseHi
searchResponse, err := search.Do(context.Background(), ds.w.client)
if err != nil {
- err = errors.Wrapf(err, "failed to search documents")
+ err = errors.Wrapf(err, "failed to opensearch documents")
return
}
@@ -197,8 +209,8 @@ func (ds *DocumentStore) Search(query Query, limit int64) (res *SearchResponseHi
var response SearchResponseBody
err = json.Unmarshal(bodyData, &response)
if err != nil {
- logs.Info("search response body: %s", string(bodyData))
- err = errors.Wrapf(err, "cannot decode search response body")
+ logs.Info("opensearch response body: %s", string(bodyData))
+ err = errors.Wrapf(err, "cannot decode opensearch response body")
return
}
@@ -210,9 +222,8 @@ func (ds *DocumentStore) Get(id string) (res *GetResponseBody, err error) {
return nil, errors.Errorf("document store == nil")
}
get := opensearchapi.GetRequest{
- Index: ds.name,
- DocumentType: "_doc",
- DocumentID: id,
+ Index: ds.name,
+ DocumentID: id,
}
getResponse, err := get.Do(context.Background(), ds.w.client)
if err != nil {
@@ -256,9 +267,8 @@ func (ds *DocumentStore) Delete(id string) (err error) {
}()
del := opensearchapi.DeleteRequest{
- Index: ds.name,
- DocumentType: "_doc",
- DocumentID: id,
+ Index: ds.name,
+ DocumentID: id,
}
delResponse, err = del.Do(context.Background(), ds.w.client)
if err != nil {
diff --git a/search/opensearch_types.go b/opensearch/opensearch_types.go
similarity index 82%
rename from search/opensearch_types.go
rename to opensearch/opensearch_types.go
index 8e52f073ab265d379c9978dcf78c460cc85c5cd0..03f14aa6d8f9efd0f0dab5ad022c73011c3a0692 100644
--- a/search/opensearch_types.go
+++ b/opensearch/opensearch_types.go
@@ -1,4 +1,4 @@
-package search
+package opensearch
// Settings
// Mapping configures an index in OpenSearch
@@ -7,8 +7,17 @@ type Settings struct {
}
type SettingsIndex struct {
- NumberOfShards int `json:"number_of_shards,omitempty"`
- NumberOfReplicas int `json:"number_of_replicas,omitempty"`
+ NumberOfShards int `json:"number_of_shards,omitempty"`
+ NumberOfReplicas int `json:"number_of_replicas,omitempty"`
+ Mapping *Mapping `json:"mapping,omitempty"`
+}
+
+type Mapping struct {
+ TotalFields TotalFields `json:"total_fields,omitempty"`
+}
+
+type TotalFields struct {
+ Limit int `json:"limit,omitempty"`
}
type Mappings struct {
@@ -36,7 +45,7 @@ type SearchRequestBody struct {
Size int64 `json:"size,omitempty"` // limit
From int64 `json:"from,omitempty"` // offset
Query Query `json:"query"`
- Timeout string `json:"timeout,omitempty"` // timeout for search
+ Timeout string `json:"timeout,omitempty"` // timeout for opensearch
}
// https://opensearch.org/docs/latest/opensearch/query-dsl/bool/
@@ -84,7 +93,7 @@ type QueryTerm map[string]string
type QueryWildcard map[string]string
type QueryMultiMatch struct {
- Query string `json:"query"` // Full value match search in selected fields
+ Query string `json:"query"` // Full value match opensearch in selected fields
Fields []string `json:"fields,omitempty" doc:"List of fields"`
Type QueryMultiMatchType `json:"type,omitempty"`
Operator string `json:"operator,omitempty"`
@@ -98,7 +107,7 @@ const (
)
type QueryString struct {
- Query string `json:"query"` // Text search with partial matches, using asterisk for optional or question mark for required wildcards before and/or after text
+ Query string `json:"query"` // Text opensearch with partial matches, using asterisk for optional or question mark for required wildcards before and/or after text
Fields []string `json:"fields,omitempty" doc:"List of fields"`
DefaultOperator string `json:"default_operator,omitempty"`
}
@@ -108,6 +117,7 @@ type QueryRange map[string]QueryExpr
type QueryExpr map[string]string
// SearchResponseBody example:
+//
// {
// "took":872,
// "timed_out":false,
@@ -124,26 +134,26 @@ type QueryExpr map[string]string
// },
// "max_score":null,
// "hits":[
-// {
-// "_index": "go-utils-audit-test-20211030",
-// "_type": "_doc",
-// "_id": "Tj9l5XwBWRiAneoYazic",
-// "_score": 1.2039728,
-// "_source": {
-// "@timestamp": "2021-10-30T15:03:20.679481+02:00",
-// "@end_time": "2021-10-30T15:03:20.469481+02:00",
-// "@duration_ms": -210,
-// "test1": "6",
-// "test2": "ACC_00098",
-// "test3": 10,
-// "http": {
-// "method": "GET",
-// "path": "/accounts"
-// },
-// "http_method": "GET",
-// "http_path": "/accounts"
-// }
-// },
+// {
+// "_index": "go-utils-audit-test-20211030",
+// "_type": "_doc",
+// "_id": "Tj9l5XwBWRiAneoYazic",
+// "_score": 1.2039728,
+// "_source": {
+// "@timestamp": "2021-10-30T15:03:20.679481+02:00",
+// "@end_time": "2021-10-30T15:03:20.469481+02:00",
+// "@duration_ms": -210,
+// "test1": "6",
+// "test2": "ACC_00098",
+// "test3": 10,
+// "http": {
+// "method": "GET",
+// "path": "/accounts"
+// },
+// "http_method": "GET",
+// "http_path": "/accounts"
+// }
+// },
// ]
// }
// }
@@ -181,16 +191,17 @@ type HitDoc struct {
}
// GetResponseBody Example:
-// {
-// "_index": "go-utils-search-docs-test",
-// "_type": "_doc",
-// "_id": "836c6443-5b0e-489b-aa0f-712ebed96841",
-// "_version": 1,
-// "_seq_no": 6,
-// "_primary_term": 1,
-// "found": true,
-// "_source": { ... }
-// }
+//
+// {
+// "_index": "go-utils-opensearch-docs-test",
+// "_type": "_doc",
+// "_id": "836c6443-5b0e-489b-aa0f-712ebed96841",
+// "_version": 1,
+// "_seq_no": 6,
+// "_primary_term": 1,
+// "found": true,
+// "_source": { ... }
+// }
type GetResponseBody struct {
Index string `json:"_index"` // name of index
Type string `json:"_type"` // _doc
diff --git a/search/time_series.go b/opensearch/time_series.go
similarity index 92%
rename from search/time_series.go
rename to opensearch/time_series.go
index 8eb4d90eea38a5a41f326e17df33370e30c9501f..54015d2628a1ae05dd91ff30060b45ea6138440d 100644
--- a/search/time_series.go
+++ b/opensearch/time_series.go
@@ -1,4 +1,4 @@
-package search
+package opensearch
import (
"bytes"
@@ -10,7 +10,7 @@ import (
"strings"
"time"
- "github.com/opensearch-project/opensearch-go/opensearchapi"
+ "github.com/opensearch-project/opensearch-go/v2/opensearchapi"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
)
@@ -37,8 +37,11 @@ type TimeSeries struct {
}
// NewTimeSeries purpose:
-// create a time series to write e.g. api api_logs
+//
+// create a time series to write e.g. api api_logs
+//
// parameters:
+//
// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-api_logs"
// the actual indices in openSearch will be called "<indexName>-<ccyymmdd>" e.g. "uafrica-v3-api-api_logs-20210102"
// tmpl must be your log data struct consisting of public fields as:
@@ -77,6 +80,7 @@ func (w *Writer) NewTimeSeries(name string, tmpl interface{}) (TimeSeries, error
Index: &SettingsIndex{
NumberOfShards: 4,
NumberOfReplicas: 0,
+ Mapping: &Mapping{TotalFields{Limit: 2000}},
},
}
@@ -130,7 +134,7 @@ func structMappingProperties(structType reflect.Type) (map[string]MappingPropert
continue
}
- // get default type of search value from field type
+ // get default type of opensearch value from field type
fieldMapping := MappingProperty{Type: "text"}
switch structField.Type.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
@@ -216,7 +220,8 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
return errors.Errorf("cannot write %T into TimeSeries(%s), expecting %s", data, ts.name, ts.dataType)
}
- // get daily search index to write to, from start time
+ // get daily opensearch index to write to, from start time
+ // indexName := ts.name + "-" + startTime.Format("20060102")
indexName := ts.name + "-" + startTime.Format("20060102")
if _, ok := ts.createdDates[indexName]; !ok {
// create new index for this date - if not exists
@@ -226,6 +231,9 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
strings.NewReader(string(ts.jsonSettings)),
)
if err != nil {
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Wrapf(err, "failed to create index(%s)", indexName)
}
switch res.StatusCode {
@@ -233,6 +241,9 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
case http.StatusCreated:
case http.StatusConflict: // 409 = already exists
default:
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Errorf("failed to create index(%s): %v %s %s", indexName, res.StatusCode, res.Status(), res.String())
}
@@ -241,6 +252,9 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
Body: strings.NewReader(string(ts.jsonMappings)),
}.Do(context.Background(), ts.w.client)
if err != nil {
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Wrapf(err, "failed to create index(%s)", indexName)
}
switch res.StatusCode {
@@ -248,6 +262,9 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
case http.StatusCreated:
case http.StatusConflict: // 409 = already exists
default:
+ if res != nil {
+ res.Body.Close()
+ }
return errors.Errorf("failed to create index(%s): %v %s %s", indexName, res.StatusCode, res.Status(), res.String())
}
ts.createdDates[indexName] = true
@@ -261,19 +278,22 @@ func (ts *TimeSeries) Write(startTime, endTime time.Time, data interface{}) erro
EndTime: endTime,
DurationMs: endTime.Sub(startTime).Milliseconds(),
}))
- if res, err := ts.w.Write(indexName, "", x.Elem().Interface()); err != nil {
+ if indexResponse, err := ts.w.Write(indexName, "", x.Elem().Interface()); err != nil {
return err
} else {
- logs.Info("IndexResponse: %+v", res)
+ logs.Info("IndexResponse: %+v", indexResponse)
}
return nil
}
// DelOldTimeSeries parameters:
-// indexName is index prefix before dash-date, e.g. "api-api_logs" then will look for "api-api_logs-<date>"
+//
+// indexName is index prefix before dash-date, e.g. "api-api_logs" then will look for "api-api_logs-<date>"
+//
// returns
-// list of indices to delete with err==nil if deleted successfully
+//
+// list of indices to delete with err==nil if deleted successfully
func (ts *TimeSeries) DelOldTimeSeries(olderThanDays int) ([]string, error) {
if olderThanDays < 0 {
return nil, errors.Errorf("invalid olderThanDays=%d < 0", olderThanDays)
@@ -349,8 +369,9 @@ type IndexSettings struct {
// Search returns docs indexed on OpenSearch document ID which cat be used in Get(id)
// The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
// So you can safely type assert e.g.
+//
// type myType struct {...}
-// ts := search.TimeSeries(..., myType{})
+// ts := opensearch.TimeSeries(..., myType{})
// docs,totalCount,err := ts.Search(...)
// if err == nil {
// for id,docValue := range docs {
@@ -368,7 +389,7 @@ func (ts *TimeSeries) Search(query Query, sort []map[string]string, limit int64,
return
}
- // example search request body for free text
+ // example opensearch request body for free text
// {
// "size": 5,
// "query": {
@@ -417,8 +438,8 @@ func (ts *TimeSeries) Search(query Query, sort []map[string]string, limit int64,
var response SearchResponseBody
err = json.Unmarshal(bodyData, &response)
if err != nil {
- logs.Info("search response body: %s", string(bodyData))
- err = errors.Wrapf(err, "cannot decode search response body")
+ logs.Info("opensearch response body: %s", string(bodyData))
+ err = errors.Wrapf(err, "cannot decode opensearch response body")
return
}
@@ -434,9 +455,8 @@ func (ts *TimeSeries) Get(id string) (res *GetResponseBody, err error) {
}
parts := strings.SplitN(id, "/", 2)
get := opensearchapi.GetRequest{
- Index: parts[0],
- DocumentType: "_doc",
- DocumentID: parts[1],
+ Index: parts[0],
+ DocumentID: parts[1],
}
getResponse, err := get.Do(context.Background(), ts.w.client)
if err != nil {
diff --git a/search/writer.go b/opensearch/writer.go
similarity index 91%
rename from search/writer.go
rename to opensearch/writer.go
index 6ba69f8dd950cd73bf02808ac4a5f44cf2095dfc..3091ebd9bd2cd1abdcd0553d16df67bd67d70fbe 100644
--- a/search/writer.go
+++ b/opensearch/writer.go
@@ -1,10 +1,10 @@
-package search
+package opensearch
import (
"crypto/tls"
"encoding/json"
- opensearch "github.com/opensearch-project/opensearch-go"
- opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
+ opensearch "github.com/opensearch-project/opensearch-go/v2"
+ opensearchapi "github.com/opensearch-project/opensearch-go/v2/opensearchapi"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
"net/http"
@@ -71,7 +71,15 @@ func (writer Writer) Write(indexName string, id string, doc interface{}) (*Index
if id != "" {
options = append(options, writer.api.Index.WithDocumentID(id))
}
- indexResponse, err := writer.api.Index(
+
+ var indexResponse *opensearchapi.Response
+ var err error
+ defer func() {
+ if indexResponse != nil {
+ indexResponse.Body.Close()
+ }
+ }()
+ indexResponse, err = writer.api.Index(
indexName,
strings.NewReader(jsonDocStr),
options...,
diff --git a/responses/responses.go b/responses/responses.go
index 0cec3b118b4f7021ac4961ba1bd5581078cd85c8..78d1d9243c28eaf364f5cfa3152aab6317a0c536 100644
--- a/responses/responses.go
+++ b/responses/responses.go
@@ -2,6 +2,7 @@ package responses
import (
"encoding/json"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/map_utils"
"net/http"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
@@ -29,7 +30,7 @@ func NotFoundResponse(err error) (events.APIGatewayProxyResponse, error) {
func NoContent() (events.APIGatewayProxyResponse, error) {
return events.APIGatewayProxyResponse{
StatusCode: http.StatusNoContent,
- Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
}, nil
}
@@ -45,7 +46,7 @@ func TooManyRequests(message string) (events.APIGatewayProxyResponse, error) {
return events.APIGatewayProxyResponse{
StatusCode: http.StatusTooManyRequests,
- Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
Body: string(responseJson),
}, nil
}
@@ -85,7 +86,7 @@ func GenericResponse(statusCode int, body string, headers map[string]string) (ev
return events.APIGatewayProxyResponse{
StatusCode: statusCode,
Body: body + "\n",
- Headers: utils.MergeMaps(utils.CorsHeaders(), headers),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), headers),
}, nil
}
@@ -101,7 +102,7 @@ func MaintenanceResponse(message string) events.APIGatewayProxyResponse {
return events.APIGatewayProxyResponse{
StatusCode: http.StatusTeapot,
Body: message,
- Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
}
}
@@ -109,13 +110,13 @@ func RateLimitResponse(message string) events.APIGatewayProxyResponse {
return events.APIGatewayProxyResponse{
StatusCode: http.StatusTooManyRequests,
Body: message,
- Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
}
}
func OptionsResponse() events.APIGatewayProxyResponse {
return events.APIGatewayProxyResponse{
StatusCode: http.StatusNoContent,
- Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Headers: map_utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
}
}
diff --git a/s3/s3.go b/s3/s3.go
index 997f8640e568c0eda68af4c2dc9fec321b06f285..7d41422739e0f2d4049e7de7964be89c9aca821b 100644
--- a/s3/s3.go
+++ b/s3/s3.go
@@ -4,7 +4,10 @@ import (
"bytes"
"encoding/binary"
"fmt"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/secrets_manager"
"net/url"
+ "os"
"path"
"strings"
"time"
@@ -78,10 +81,55 @@ const (
MIMETypeXLSX MIMEType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
+var (
+ sessions = map[string]*SessionWithHelpers{}
+)
+
type SessionWithHelpers struct {
S3Session *s3.S3
}
+func GetSession(isDebug bool, region ...string) *SessionWithHelpers {
+ s3Region := os.Getenv("AWS_REGION")
+
+ // Set custom region
+ if region != nil && len(region) > 0 {
+ s3Region = region[0]
+ }
+
+ // Check if session exists for region, if it does return it
+ if s3Session, ok := sessions[s3Region]; ok {
+ return s3Session
+ }
+
+ // Setup session
+ s3Credentials := GetS3SessionCredentials(isDebug)
+ options := session.Options{
+ Config: aws.Config{
+ Region: aws.String(s3Region),
+ Credentials: s3Credentials,
+ },
+ }
+
+ sess, err := session.NewSessionWithOptions(options)
+ if err != nil {
+ return nil
+ }
+
+ s3Session := NewSession(sess)
+ sessions[s3Region] = s3Session
+ return s3Session
+}
+
+func GetS3SessionCredentials(isDebug bool) *credentials.Credentials {
+ secretID := os.Getenv("S3_SECRET_ID")
+ s3Credentials, err := secrets_manager.GetS3UploadCredentials(secretID, isDebug)
+ if err != nil {
+ return nil
+ }
+ return s3Credentials
+}
+
func NewSession(session *session.Session) *SessionWithHelpers {
return &SessionWithHelpers{
S3Session: s3.New(session),
diff --git a/slice_utils/slice_utils.go b/slice_utils/slice_utils.go
index 0c400cb7863bf42ead48dfee4ee6b57464347de5..f203d0dc8d0dd698f63a5e53a815a990fb005aff 100644
--- a/slice_utils/slice_utils.go
+++ b/slice_utils/slice_utils.go
@@ -53,3 +53,14 @@ func FilterNonEmptyString(arr []string) []string {
}).([]string)
return nonEmptyStrings
}
+
+func ArraySlice(s []any, offset, length int) []any {
+ if offset > len(s) {
+ return s
+ }
+ end := offset + length
+ if end < len(s) {
+ return s[offset:end]
+ }
+ return s[offset:]
+}
diff --git a/sqs/sqs.go b/sqs/sqs.go
index 3275a67fd834dac6855f6d427f22bfbcd93f5380..369c220906d5da06d7844d8999d564d4674cfd21 100644
--- a/sqs/sqs.go
+++ b/sqs/sqs.go
@@ -14,6 +14,7 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sqs"
"gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/logs"
+ "github.com/go-resty/resty/v2"
)
var sqsClient *sqs.SQS
@@ -102,7 +103,19 @@ func (m *Messenger) SendSQSMessage(headers map[string]string, body string, curre
return *res.MessageId, err
}
-func SendSQSMessage(msgr Messenger, objectToSend interface{}, currentRequestID *string, sqsType string) error {
+func SendSQSMessage(msgr Messenger, objectToSend interface{}, currentRequestID *string, sqsType string, isDebug bool) error {
+
+ if isDebug {
+ go func() {
+ resty.New().R().
+ SetBody(objectToSend).
+ Post("http://127.0.0.1:3000/sqs/" + sqsType)
+ }()
+ time.Sleep(time.Second*1)
+ return nil
+ }
+
+
if sqsClient == nil {
var err error
sqsClient, err = NewSQSClient(msgr.Region)
diff --git a/string_utils/string_utils.go b/string_utils/string_utils.go
index 67bd3de4a71ef1ccdaa526bcc175ef2805841112..995e94a309eabbf18f1bee3cd9462242d930a2c4 100644
--- a/string_utils/string_utils.go
+++ b/string_utils/string_utils.go
@@ -1,9 +1,10 @@
package string_utils
import (
+ "bytes"
"encoding/json"
"fmt"
- "net/url"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/number_utils"
"regexp"
"strconv"
"strings"
@@ -65,31 +66,6 @@ func IsNumericString(s string) bool {
return err == nil
}
-// StandardisePhoneNumber standardises phone numbers with +27 instead of 0 prefix
-func StandardisePhoneNumber(number string) string {
- // is the first rune/char of the string a 0
- if []rune(number)[0] == []rune("0")[0] {
- // Add south african country code (hardcoded for now)
- number = "+27" + number[1:]
- }
- return number
-}
-
-func FormatPhoneNumber(phoneNumber string) string {
- if len(phoneNumber) > 7 {
- return phoneNumber
- }
-
- // Format as 076 453 2188
- phoneNumber = insertInto(phoneNumber, 3, " ")
- phoneNumber = insertInto(phoneNumber, 7, " ")
- return phoneNumber
-}
-
-func insertInto(s string, index int, characters string) string {
- return s[:index] + characters + s[index:]
-}
-
func IsAlphaNumeric(str string) bool {
regex := regexp.MustCompile("^[a-zA-Z0-9]*$")
return regex.MatchString(str)
@@ -100,22 +76,10 @@ func IsAlphaNumericOrDash(str string) bool {
return regex.MatchString(str)
}
-func IsValidUsername(str string) bool {
- regex := regexp.MustCompile("^[a-zA-Z0-9-.@+_]*$")
- return regex.MatchString(str)
-}
-
func Equal(a string, b string) bool {
return strings.TrimSpace(strings.ToLower(a)) == strings.TrimSpace(strings.ToLower(b))
}
-func UnwrapString(s *string) string {
- if s == nil {
- return ""
- }
- return *s
-}
-
// TrimP trims specified strings, replacing empty string with nil
func TrimP(sp *string) *string {
if sp == nil {
@@ -142,6 +106,20 @@ func ConcatP(args ...*string) string {
return s
}
+// Concat concatenates all specified non-empty strings with ", " separators
+func Concat(args []string, separator string) string {
+ s := ""
+ for _, arg := range args {
+ if arg != "" {
+ if s != "" {
+ s += separator
+ }
+ s += arg
+ }
+ }
+ return s
+}
+
func ToJSONString(object interface{}) (string, error) {
jsonBytes, err := json.Marshal(&object)
if err != nil {
@@ -157,9 +135,6 @@ func Int64ToString(number int64) string {
func IntToString(number int) string {
return strconv.Itoa(number)
}
-func StringToInt(stringValue string) (int, error) {
- return strconv.Atoi(stringValue)
-}
func Int64SliceToString(numbers []int64) string {
numString := fmt.Sprint(numbers)
@@ -175,19 +150,6 @@ func Int64SliceToStringSlice(numbers []int64) []string {
return numStringSlice
}
-func StringToInt64(stringValue string) (int64, error) {
- number, err := strconv.ParseInt(stringValue, 10, 64)
- return number, err
-}
-
-func StringToFloat64(stringValue string) (float64, error) {
- number, err := strconv.ParseFloat(stringValue, 64)
- if err != nil {
- return 0, err
- }
- return number, nil
-}
-
func Float64ToString(number float64, precision int) string {
return strconv.FormatFloat(number, 'f', precision, 64)
}
@@ -197,18 +159,10 @@ func Float64ToStringWithPrec(number float64, prec int) string {
}
func ValidateStringAsInt64(stringValue string) error {
- _, err := StringToInt64(stringValue)
+ _, err := number_utils.StringToInt64(stringValue)
return err
}
-func PtoString(stringPointer *string) string {
- if stringPointer == nil {
- return ""
- }
-
- return *stringPointer
-}
-
func IsEmpty(sp *string) bool {
if sp == nil {
return true
@@ -246,35 +200,6 @@ func SentenceCase(str string) string {
return ""
}
-// RemoveUrlScheme Removes http:// or https:// from a URL
-func RemoveUrlScheme(str string) string {
- newStr := strings.Replace(str, "http://", "", 1)
- newStr = strings.Replace(str, "https://", "", 1)
- return newStr
-}
-
-// EscapeOpenSearchSearchString See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters
-func EscapeOpenSearchSearchString(str string) string {
- searchString := str
-
- // Reserved characters
- // NOTE: first char must be "\" to prevent replacing it again after replacing other chars with "\\"
- reservedCharacters := []string{"\\", "+", "-", "=", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", "\"", "~", "*", "?", ":", "/"}
-
- // Remove "<" and ">"
- strings.ReplaceAll(searchString, "<", "")
- strings.ReplaceAll(searchString, ">", "")
-
- // Escape the reserved characters with double backslashes ("\\")
- for _, char := range reservedCharacters {
- if strings.Contains(searchString, char) {
- re := regexp.MustCompile(char)
- searchString = re.ReplaceAllString(searchString, "\\"+char)
- }
- }
- return searchString
-}
-
// SplitString separates a string on any character in the list of sep
func SplitString(str string, sep []rune) []string {
splitStrings := strings.FieldsFunc(str, func(c rune) bool {
@@ -284,12 +209,6 @@ func SplitString(str string, sep []rune) []string {
return splitStrings
}
-// IsUrlStrict Returns whether a URL is valid in a strict way (Must have scheme and host)
-func IsUrlStrict(str string) bool {
- u, err := url.Parse(str)
- return err == nil && u.Scheme != "" && u.Host != ""
-}
-
func LimitStringToMaxLength(str string, maxLen int) string {
if len(str) > maxLen {
str = str[0:maxLen]
@@ -297,23 +216,6 @@ func LimitStringToMaxLength(str string, maxLen int) string {
return str
}
-// StripQueryString - Strips the query parameters from a URL
-func StripQueryString(inputUrl string) (string, error) {
- u, err := url.Parse(inputUrl)
- if err != nil {
- return inputUrl, err
- }
- u.RawQuery = ""
- return u.String(), nil
-}
-
-func IsValidEmail(email string) bool {
- // Found here: https://stackoverflow.com/questions/201323/how-can-i-validate-an-email-address-using-a-regular-expression
- regex, _ := regexp.Compile("(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])")
- isValidEmail := regex.MatchString(email)
- return isValidEmail
-}
-
func PascalCaseToSentence(pascal string) string {
var parts []string
start := 0
@@ -330,3 +232,16 @@ func PascalCaseToSentence(pascal string) string {
return sentence
}
+
+func PrettyJSON(jsonString string) (validJson bool, prettyString string) {
+ var prettyJSON bytes.Buffer
+ err := json.Indent(&prettyJSON, []byte(jsonString), "", " ")
+ if err != nil {
+ validJson = false
+ prettyString = jsonString
+ } else {
+ validJson = true
+ prettyString = prettyJSON.String()
+ }
+ return
+}
diff --git a/struct_utils/struct_utils.go b/struct_utils/struct_utils.go
new file mode 100644
index 0000000000000000000000000000000000000000..384feb3cc58b35d2bab4e3661f285d161bfb41a2
--- /dev/null
+++ b/struct_utils/struct_utils.go
@@ -0,0 +1,39 @@
+package struct_utils
+
+import "strings"
+
+// KeyValuePair defines a key/value pair derived from form data
+type KeyValuePair struct {
+ Key string `json:"key"`
+ Value string `json:"value"`
+}
+
+// FormToKeyValuePairs returns a string-based map of strings as derived from posted form keys and values.
+// e.g. oauth_consumer_key=mlhgs&oauth_consumer_secret=x240ar&oauth_verifier=b0qjbx&store_base_url=http%3A%2F%2Flocalhost.com%2Fstore
+func FormToKeyValuePairs(body string) []KeyValuePair {
+ out := []KeyValuePair{}
+ parts := strings.Split(body, "&")
+ for _, p := range parts {
+ split := strings.Split(p, "=")
+ k := split[0]
+ v := split[1]
+ kv := KeyValuePair{
+ Key: k,
+ Value: v,
+ }
+ out = append(out, kv)
+ }
+
+ return out
+}
+
+// GetValue returns the value for the given key from a KeyValuePair slice.
+func GetValue(key string, kv []KeyValuePair) string {
+ for _, v := range kv {
+ if v.Key == key {
+ return v.Value
+ }
+ }
+
+ return ""
+}
diff --git a/utils/utils.go b/utils/utils.go
index f859bd0fda5cf3a6729bb64dd53ce07da6e4054d..b537f8163ddf17b48b95a2cd6565e0aed13c210e 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -3,15 +3,17 @@ package utils
import (
"archive/zip"
"bytes"
- "encoding/json"
- "fmt"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors"
+ "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/string_utils"
"io/ioutil"
+ "net/mail"
+ "net/url"
"os"
- "strconv"
+ "regexp"
"strings"
"time"
- "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/struct_utils"
+ "github.com/mohae/deepcopy"
)
// GetEnv is a helper function for getting environment variables with a default
@@ -98,67 +100,116 @@ func readZipFile(zipFile *zip.File) ([]byte, error) {
return ioutil.ReadAll(zipFileData)
}
-func DeepCopy(toValue interface{}, fromValue interface{}) (err error) {
- valueBytes, err := json.Marshal(fromValue)
- if err != nil {
- return err
+func DeepCopy(fromValue interface{}) (toValue interface{}) {
+ return deepcopy.Copy(fromValue)
+}
+
+func ValueToPointer[V any](value V) *V {
+ return &value
+}
+
+func PointerToValue[V any](value *V) V {
+ if value != nil {
+ return *value
}
- err = struct_utils.UnmarshalJSON(valueBytes, toValue)
+
+ return *new(V) // zero value of V
+}
+
+func ValidateEmailAddress(email string) (string, error) {
+ if email == "" {
+ return "", errors.Error("email address is empty")
+ }
+
+ cleanEmail := strings.ToLower(strings.TrimSpace(email))
+ cleanEmail = string_utils.RemoveAllWhiteSpaces(cleanEmail)
+
+ // We validate it but still return it since in some cases we don't want to break everything if the email is bad
+ _, err := mail.ParseAddress(cleanEmail)
if err != nil {
- return err
- }
- return nil
-}
-
-func UnwrapBool(b *bool) bool {
- if b == nil {
- return false
- }
- return *b
-}
-
-// MapStringInterfaceToMapStringString converts a generic value typed map to a map with string values
-func MapStringInterfaceToMapStringString(inputMap map[string]interface{}) map[string]string {
- query := make(map[string]string)
- for mapKey, mapVal := range inputMap {
- // Check if mapVal is a slice or a single value
- switch mapValTyped := mapVal.(type) {
- case []interface{}:
- // Slice - convert each element individually
- var mapValString []string
-
- // Loop through each element in the slice and check the type
- for _, sliceElem := range mapValTyped {
- switch sliceElemTyped := sliceElem.(type) {
- case string:
- // Enclose strings in escaped quotations
- mapValString = append(mapValString, fmt.Sprintf("\"%v\"", sliceElemTyped))
- case float64:
- // Use FormatFloat for least amount of precision.
- mapValString = append(mapValString, strconv.FormatFloat(sliceElemTyped, 'f', -1, 64))
- default:
- // Convert to string
- mapValString = append(mapValString, fmt.Sprintf("%v", sliceElemTyped))
- }
- }
- // Join as a comma seperated array
- query[mapKey] = "[" + strings.Join(mapValString, ",") + "]"
- default:
- // Single value - convert to string
- query[mapKey] = fmt.Sprintf("%v", mapVal)
- }
+ return cleanEmail, errors.Wrap(err, "could not parse email address")
}
- return query
+ return cleanEmail, nil
}
-// MergeMaps If there are similar properties in the maps, the last one will be used as the value
-func MergeMaps(maps ...map[string]string) map[string]string {
- ret := map[string]string{}
- for _, mapV := range maps {
- for k, v := range mapV {
- ret[k] = v
+// IsUrlStrict Returns whether a URL is valid in a strict way (Must have scheme and host)
+func IsUrlStrict(str string) bool {
+ u, err := url.Parse(str)
+ return err == nil && u.Scheme != "" && u.Host != ""
+}
+
+func IsValidUsername(str string) bool {
+ regex := regexp.MustCompile("^[a-zA-Z0-9-.@+_]*$")
+ return regex.MatchString(str)
+}
+
+// StandardisePhoneNumber standardises phone numbers with +27 instead of 0 prefix
+func StandardisePhoneNumber(number string) string {
+ number = strings.TrimSpace(number)
+
+ if number == "" {
+ return number
+ }
+
+ // is the first rune/char of the string a 0
+ if []rune(number)[0] == []rune("0")[0] {
+ // Add south african country code (hardcoded for now)
+ number = "+27" + number[1:]
+ }
+ return number
+}
+
+func FormatPhoneNumber(phoneNumber string) string {
+ if len(phoneNumber) > 7 {
+ return phoneNumber
+ }
+
+ // Format as 076 453 2188
+ phoneNumber = insertInto(phoneNumber, 3, " ")
+ phoneNumber = insertInto(phoneNumber, 7, " ")
+ return phoneNumber
+}
+
+func insertInto(s string, index int, characters string) string {
+ return s[:index] + characters + s[index:]
+}
+
+// RemoveUrlScheme Removes http:// or https:// from a URL
+func RemoveUrlScheme(str string) string {
+ newStr := strings.Replace(str, "http://", "", 1)
+ newStr = strings.Replace(str, "https://", "", 1)
+ return newStr
+}
+
+// StripQueryString - Strips the query parameters from a URL
+func StripQueryString(inputUrl string) (string, error) {
+ u, err := url.Parse(inputUrl)
+ if err != nil {
+ return inputUrl, err
+ }
+ u.RawQuery = ""
+ return u.String(), nil
+}
+
+// EscapeOpenSearchSearchString See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#_reserved_characters
+func EscapeOpenSearchSearchString(str string) string {
+ searchString := str
+
+ // Reserved characters
+ // NOTE: first char must be "\" to prevent replacing it again after replacing other chars with "\\"
+ reservedCharacters := []string{"\\", "+", "-", "=", "&&", "||", "!", "(", ")", "{", "}", "[", "]", "^", "\"", "~", "*", "?", ":", "/"}
+
+ // Remove "<" and ">"
+ strings.ReplaceAll(searchString, "<", "")
+ strings.ReplaceAll(searchString, ">", "")
+
+ // Escape the reserved characters with double backslashes ("\\")
+ for _, char := range reservedCharacters {
+ if strings.Contains(searchString, char) {
+ re := regexp.MustCompile(char)
+ searchString = re.ReplaceAllString(searchString, "\\"+char)
}
}
- return ret
+ return searchString
}