From ea83f988e995c2a3d447d50c3b66d2eceb7c7c42 Mon Sep 17 00:00:00 2001
From: Jan Semmelink <jan@uafrica.com>
Date: Thu, 11 Nov 2021 10:50:23 +0200
Subject: [PATCH] Update config to allow load from ENV and override from REDIS
for dynamic values
---
config/struct.go | 43 +++++++++++++++++++++++++-
config/struct_test.go | 4 +--
redis/redis.go | 34 ++++++++++++++++----
struct_utils/named_values_to_struct.go | 40 ++++++++++++++----------
4 files changed, 96 insertions(+), 25 deletions(-)
diff --git a/config/struct.go b/config/struct.go
index 84053ad..04bfb88 100644
--- a/config/struct.go
+++ b/config/struct.go
@@ -5,6 +5,7 @@ import (
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/string_utils"
"gitlab.com/uafrica/go-utils/struct_utils"
)
@@ -12,7 +13,47 @@ var (
prefixStructs = map[string]interface{}{}
)
-func Load(prefix string, configStructPtr interface{}) error {
+func LoadEnv(prefix string, configStructPtr interface{}) error {
+ return Load(prefix, configStructPtr, string_utils.EnvironmentKeyReader())
+}
+
+func Load(prefix string, configStructPtr interface{}, keyReader string_utils.KeyReader) error {
+ if !prefixRegex.MatchString(prefix) {
+ return errors.Errorf("config(%s) invalid prefix", prefix)
+ }
+
+ //store before load in case it fails to be still part of docs
+ prefixStructs[prefix] = configStructPtr
+
+ //read os.Getenv() or other reader...
+ nv := struct_utils.NamedValuesFromReader(prefix, keyReader)
+ logger.Debugf("nv: %+v", nv)
+
+ //parse into struct
+ unused, err := struct_utils.UnmarshalNamedValues(nv, configStructPtr)
+ if err != nil {
+ return errors.Wrapf(err, "config(%s) cannot load", prefix)
+ }
+ if len(unused) > 0 {
+ //we still use os.Getenv() elsewhere, so some variables may not be in the struct
+ //e.g. AUDIT_QUEUE_URL is read from queues/sqs/producer which match config(prefix="AUDIT")
+ //so we cannot yet fail here, which we should, because config setting not used is often
+ //a reason for errors, when we try to configure something, then it does not work, and
+ //we cannot figure out why, but the value we did set, might just be misspelled etc.
+ //so, for now - do not fail here, just report the unused values
+ logger.Warnf("Note unused env (might be used elsewhere) for config(%s): %+v", prefix, unused)
+ //return errors.Errorf("config(%s): unknown %+v", prefix, unused)
+ }
+
+ if validator, ok := configStructPtr.(Validator); ok {
+ if err := validator.Validate(); err != nil {
+ return errors.Wrapf(err, "config(%s) is invalid", prefix)
+ }
+ }
+ return nil
+}
+
+func LoadRedis(prefix string, configStructPtr interface{}) error {
if !prefixRegex.MatchString(prefix) {
return errors.Errorf("config(%s) invalid prefix", prefix)
}
diff --git a/config/struct_test.go b/config/struct_test.go
index 587f8bd..297664f 100644
--- a/config/struct_test.go
+++ b/config/struct_test.go
@@ -37,7 +37,7 @@ func TestLoad(t *testing.T) {
os.Setenv("TEST_VALUE_HOLIDAYS", "[2021-03-21,2021-04-27,2021-05-01,2021-06-16,2021-08-09,2021-12-16,2021-12-25]")
c := Config{}
- if err := config.Load("TEST_VALUE", &c); err != nil {
+ if err := config.LoadEnv("TEST_VALUE", &c); err != nil {
t.Fatalf("Cannot load config: %+v", err)
}
t.Logf("Loaded config: %+v", c)
@@ -135,7 +135,7 @@ func TestLogConfig(t *testing.T) {
os.Setenv("LOG_OTHER", "1")
os.Setenv("LOG_SEARCH_OTHER", "2")
c := LogConfig{}
- err := config.Load("LOG", &c)
+ err := config.LoadEnv("LOG", &c)
if err != nil {
t.Fatalf("Failed: %+v", err)
}
diff --git a/redis/redis.go b/redis/redis.go
index 23d667e..5dac499 100644
--- a/redis/redis.go
+++ b/redis/redis.go
@@ -10,9 +10,11 @@ import (
"github.com/go-redis/redis/v8"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
type IRedis interface {
+ string_utils.KeyReader
Del(key string) error
SetJSON(key string, value interface{}) error
SetJSONIndefinitely(key string, value interface{}) error
@@ -21,7 +23,6 @@ type IRedis interface {
SetString(key string, value string) error
SetStringIndefinitely(key string, value string) error
SetStringForDur(key string, value string, dur time.Duration) error
- GetString(key string) (value string, ok bool)
}
type redisWithContext struct {
@@ -67,14 +68,18 @@ func (r redisWithContext) SetJSONForDur(key string, value interface{}, dur time.
if r.client == nil {
return errors.Errorf("REDIS disabled: cannot set JSON key(%s) = (%T)%v", key, value, value)
}
- jsonBytes, err := json.Marshal(value)
- if err != nil {
- return errors.Wrapf(err, "failed to JSON encode key(%s) = (%T)", key, value)
+ valueStr, ok := value.(string)
+ if !ok {
+ jsonBytes, err := json.Marshal(value)
+ if err != nil {
+ return errors.Wrapf(err, "failed to JSON encode key(%s) = (%T)", key, value)
+ }
+ valueStr = string(jsonBytes)
}
- if _, err = r.client.Set(r.Context, key, string(jsonBytes), dur).Result(); err != nil {
+ if _, err := r.client.Set(r.Context, key, valueStr, dur).Result(); err != nil {
return errors.Wrapf(err, "failed to set JSON key(%s)", key)
}
- logger.Debugf("REDIS.SetJSON(%s)=%s (%T) (exp: %v)", key, string(jsonBytes), value, dur)
+ logger.Debugf("REDIS.SetJSON(%s)=%s (%T) (exp: %v)", key, valueStr, value, dur)
return nil
}
@@ -130,6 +135,23 @@ func (r redisWithContext) GetString(key string) (string, bool) {
return value, true
}
+func (r redisWithContext) Keys(prefix string) []string {
+ if r.client == nil {
+ return nil
+ }
+ value, err := r.client.Keys(r.Context, prefix+"*").Result()
+ if err != nil { /* Actual error */
+ if err != redis.Nil { /* other than no keys match */
+ logger.Errorf("Error fetching redis keys(%s*): %+v", prefix, err)
+ } else {
+ logger.Errorf("Failed: %+v", err)
+ }
+ return nil //no matches
+ }
+ logger.Debugf("Keys(%s): %+v", prefix, value)
+ return value
+}
+
//global connection to REDIS used in all context
var globalClient *redis.Client
diff --git a/struct_utils/named_values_to_struct.go b/struct_utils/named_values_to_struct.go
index 2f118e4..81c5836 100644
--- a/struct_utils/named_values_to_struct.go
+++ b/struct_utils/named_values_to_struct.go
@@ -3,13 +3,14 @@ package struct_utils
import (
"encoding/csv"
"encoding/json"
- "os"
"reflect"
"sort"
"strconv"
"strings"
"gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
//Purpose:
@@ -34,24 +35,31 @@ import (
//
// MY_LIB_CONFIG_ADDRS=["55 Crescent, Town", "12 Big Street, City"] -> 2 values including commas because of quoted CSV
func NamedValuesFromEnv(prefix string) map[string][]string {
+ return NamedValuesFromReader(prefix, string_utils.EnvironmentKeyReader())
+}
+
+func NamedValuesFromReader(prefix string, reader string_utils.KeyReader) map[string][]string {
+ if reader == nil {
+ return nil
+ }
result := map[string][]string{}
prefix += "_"
- for _, env := range os.Environ() {
- if strings.HasPrefix(env, prefix) {
- parts := strings.SplitN(env[len(prefix):], "=", 2)
- if len(parts) == 2 {
- //name = parts[0], value = parts[1]
- value := parts[1]
- result[strings.ToLower(parts[0])] = []string{value}
+ for _, key := range reader.Keys(prefix) {
+ value, ok := reader.GetString(key)
+ key = key[len(prefix):]
+ if !ok {
+ logger.Debugf("Key(%s) undefined", key)
+ continue
+ }
+ logger.Debugf("key(%s)=\"%s\"", key, value)
+ result[strings.ToLower(key)] = []string{value}
- //split only if valid CSV between [...]
- if value[0] == '[' && value[len(value)-1] == ']' {
- csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
- csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
- if csvErr == nil {
- result[strings.ToLower(parts[0])] = csvValues
- }
- }
+ //split only if valid CSV between [...]
+ if value[0] == '[' && value[len(value)-1] == ']' {
+ csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
+ csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
+ if csvErr == nil {
+ result[strings.ToLower(key)] = csvValues
}
}
}
--
GitLab