From 6d165692e18632e449c05409502d68497d18f627 Mon Sep 17 00:00:00 2001
From: Jan Semmelink <jan@uafrica.com>
Date: Mon, 8 Nov 2021 09:17:55 +0200
Subject: [PATCH] Update api and config to use the same function for parsing
string values from ENV/URL into a struct
---
api/context.go | 105 +------
config/doc.go | 136 +++++++++
config/doc_example.md | 11 +
config/struct.go | 126 ++-------
config/struct_test.go | 160 ++++++++---
struct_utils/named_values_to_struct.go | 294 ++++++++++++++++++++
struct_utils/named_values_to_struct_test.go | 210 ++++++++++++++
7 files changed, 811 insertions(+), 231 deletions(-)
create mode 100644 config/doc.go
create mode 100644 config/doc_example.md
create mode 100644 struct_utils/named_values_to_struct.go
create mode 100644 struct_utils/named_values_to_struct_test.go
diff --git a/api/context.go b/api/context.go
index eea6c5c..31253eb 100644
--- a/api/context.go
+++ b/api/context.go
@@ -1,15 +1,15 @@
package api
import (
- "encoding/csv"
"encoding/json"
"reflect"
- "strings"
"github.com/aws/aws-lambda-go/events"
"gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logger"
"gitlab.com/uafrica/go-utils/reflection"
"gitlab.com/uafrica/go-utils/service"
+ "gitlab.com/uafrica/go-utils/struct_utils"
)
type Context interface {
@@ -20,8 +20,6 @@ type Context interface {
LogAPIRequestAndResponse(res events.APIGatewayProxyResponse, err error)
}
-var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
-
type apiContext struct {
service.Context
request events.APIGatewayProxyRequest
@@ -59,8 +57,13 @@ func (ctx *apiContext) LogAPIRequestAndResponse(res events.APIGatewayProxyRespon
//allocate struct for params, populate it from the URL parameters then validate and return the struct
func (ctx apiContext) GetRequestParams(paramsStructType reflect.Type) (interface{}, error) {
paramsStructValuePtr := reflect.New(paramsStructType)
- if err := ctx.setParamsInStruct("params", paramsStructType, paramsStructValuePtr.Elem()); err != nil {
- return nil, errors.Wrapf(err, "failed to put query param values into struct")
+ nv := struct_utils.NamedValuesFromURL(ctx.request.QueryStringParameters, ctx.request.MultiValueQueryStringParameters)
+ unused, err := struct_utils.UnmarshalNamedValues(nv, paramsStructValuePtr.Interface())
+ if err != nil {
+ return nil, errors.Wrapf(err, "invalid parameters")
+ }
+ if len(unused) > 0 {
+ logger.Warnf("Unknown parameters: %+v", unused)
}
if err := ctx.applyClaim("params", paramsStructValuePtr.Interface()); err != nil {
return nil, errors.Wrapf(err, "failed to fill claims on params")
@@ -73,96 +76,6 @@ func (ctx apiContext) GetRequestParams(paramsStructType reflect.Type) (interface
return paramsStructValuePtr.Elem().Interface(), nil
}
-//extract params into a struct value
-func (ctx apiContext) setParamsInStruct(name string, t reflect.Type, v reflect.Value) error {
- for i := 0; i < t.NumField(); i++ {
- tf := t.Field(i)
- //enter into anonymous sub-structs
- if tf.Anonymous {
- if tf.Type.Kind() == reflect.Struct {
- if err := ctx.setParamsInStruct(name+"."+tf.Name, t.Field(i).Type, v.Field(i)); err != nil {
- return errors.Wrapf(err, "failed on parameters %s.%s", name, tf.Name)
- }
- continue
- }
- return errors.Errorf("parameters cannot parse into anonymous %s field %s", tf.Type.Kind(), tf.Type.Name())
- }
-
- //named field:
- //use name from json tag, else lowercase of field name
- n := (strings.SplitN(tf.Tag.Get("json"), ",", 2))[0]
- if n == "" {
- n = strings.ToLower(tf.Name)
- }
- if n == "" || n == "-" {
- continue //skip fields without name
- }
-
- //see if this named param was specified
- var paramStrValues []string
- if paramStrValue, isDefined := ctx.request.QueryStringParameters[n]; isDefined {
- //specified once in URL
- if len(paramStrValue) >= 2 && paramStrValue[0] == '[' && paramStrValue[len(paramStrValue)-1] == ']' {
- //specified as CSV inside [...] e.g. id=[1,2,3]
- csvReader := csv.NewReader(strings.NewReader(paramStrValue[1 : len(paramStrValue)-1]))
- var err error
- paramStrValues, err = csvReader.Read()
- if err != nil {
- return errors.Wrapf(err, "invalid CSV: [%s]", paramStrValue)
- }
- } else {
- //specified as single value only e.g. id=1
- paramStrValues = []string{paramStrValue}
- }
- } else {
- //specified multiple times e.g. id=1&id=2&id=3
- paramStrValues = ctx.request.MultiValueQueryStringParameters[n]
- }
- if len(paramStrValues) == 0 {
- continue //param has no value specified in URL
- }
-
- valueField := v.Field(i)
- if valueField.Kind() == reflect.Ptr {
- valueField.Set(reflect.New(valueField.Type().Elem()))
- valueField = valueField.Elem()
- }
-
- //param is defined >=1 times in URL
- if tf.Type.Kind() == reflect.Slice {
- //this param struct field is a slice, iterate over all specified values
- for i, paramStrValue := range paramStrValues {
- paramValue, err := parseParamValue(paramStrValue, tf.Type.Elem())
- if err != nil {
- return errors.Wrapf(err, "invalid %s[%d]", n, i)
- }
- valueField.Set(reflect.Append(valueField, paramValue))
- }
- } else {
- if len(paramStrValues) > 1 {
- return errors.Errorf("parameter %s does not support multiple values [%s]", n, strings.Join(paramStrValues, ","))
- }
- //single value specified
- paramValue, err := parseParamValue(paramStrValues[0], valueField.Type())
- if err != nil {
- return errors.Wrapf(err, "invalid %s", n)
- }
- valueField.Set(paramValue)
- }
- } //for each param struct field
- return nil
-}
-
-func parseParamValue(s string, t reflect.Type) (reflect.Value, error) {
- newValuePtr := reflect.New(t)
- if err := json.Unmarshal([]byte("\""+s+"\""), newValuePtr.Interface()); err != nil {
- if err := json.Unmarshal([]byte(s), newValuePtr.Interface()); err != nil {
- return newValuePtr.Elem(), errors.Wrapf(err, "invalid \"%s\"", s)
- }
- }
- return newValuePtr.Elem(), nil
-}
-
func (ctx apiContext) GetRequestBody(requestStructType reflect.Type) (interface{}, error) {
requestStructValuePtr := reflect.New(requestStructType)
err := json.Unmarshal([]byte(ctx.request.Body), requestStructValuePtr.Interface())
diff --git a/config/doc.go b/config/doc.go
new file mode 100644
index 0000000..2b802b4
--- /dev/null
+++ b/config/doc.go
@@ -0,0 +1,136 @@
+package config
+
+import (
+ "fmt"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/struct_utils"
+)
+
+func Doc(f *os.File, showValues bool, format int) {
+ if f == nil {
+ return
+ }
+
+ entries := []docEntry{}
+ for prefix, structPtr := range prefixStructs {
+ prefixEntries := docStruct(prefix, reflect.TypeOf(structPtr).Elem(), reflect.ValueOf(structPtr).Elem())
+ if showValues {
+ nv := struct_utils.NamedValuesFromEnv(prefix)
+ for i, e := range prefixEntries {
+ name := strings.ToLower(e.Env[len(prefix)+1:])
+ if values, ok := nv[name]; ok {
+ e.Current = values
+ prefixEntries[i] = e
+ delete(nv, name)
+ }
+ }
+ }
+ entries = append(entries, prefixEntries...)
+ }
+
+ sort.Slice(entries, func(i, j int) bool { return entries[i].Env < entries[j].Env })
+
+ switch format {
+ case 1: //Mark Down
+ fmt.Fprintf(f, "# Configuration from Environment\n")
+ fmt.Fprintf(f, "\n")
+ if !showValues {
+ fmt.Fprintf(f, "|Environment|Type|Default|Description & Rules|\n")
+ fmt.Fprintf(f, "|---|---|---|---|\n")
+ } else {
+ fmt.Fprintf(f, "|Environment|Type|Default|Description & Rules|Values|\n")
+ fmt.Fprintf(f, "|---|---|---|---|---|\n")
+ }
+ for _, e := range entries {
+ text := e.Text
+ if text != "" && e.Rules != "" {
+ text += "; " + e.Rules
+ }
+ fmt.Fprintf(f, "|%s|%s|%s|%s|",
+ e.Env,
+ e.Type,
+ e.Default,
+ text)
+ if showValues {
+ if len(e.Current) == 0 {
+ fmt.Fprintf(f, "(Not Defined)|") //no values
+ } else {
+ if len(e.Current) == 1 {
+ fmt.Fprintf(f, "%s|", e.Current[0]) //only one value
+ } else {
+ fmt.Fprintf(f, "%s|", strings.Join(e.Current, ", ")) //multiple values
+ }
+ }
+ }
+ fmt.Fprintf(f, "\n")
+ }
+
+ default:
+ //just dump it
+ fmt.Fprintf(f, "=====[ CONFIGURATION ]=====\n")
+ for _, e := range entries {
+ fmt.Fprintf(f, "%+v\n", e)
+ }
+ }
+}
+
+func docStruct(prefix string, t reflect.Type, v reflect.Value) (entries []docEntry) {
+ logger.Debugf("docStruct(%s, %s)", prefix, t.Name())
+ entries = []docEntry{}
+ for i := 0; i < t.NumField(); i++ {
+ tf := t.Field(i)
+ if tf.Anonymous {
+ if tf.Type.Kind() == reflect.Struct {
+ entries = append(entries, docStruct(prefix, tf.Type, v.Field(i))...) //anonymous embedded sub-struct
+ }
+ continue //anonymous embedded non-struct
+ }
+
+ tag := strings.SplitN(tf.Tag.Get("json"), ",", 2)[0]
+ if tag == "" || tag == "-" {
+ continue //excluded field
+ }
+
+ fieldName := prefix + "_" + strings.ToUpper(tag)
+ switch tf.Type.Kind() {
+ case reflect.Struct:
+ entries = append(entries, docStruct(fieldName, tf.Type, v.Field(i))...) //anonymous embedded sub-struct
+
+ case reflect.Slice:
+ entries = append(entries, docEntry{
+ Env: fieldName,
+ Type: "list of " + tf.Type.Elem().Name(),
+ Text: tf.Tag.Get("doc"),
+ Default: tf.Tag.Get("default"),
+ Rules: tf.Tag.Get("rules"),
+ Value: v.Field(i),
+ })
+
+ default:
+ entries = append(entries, docEntry{
+ Env: fieldName,
+ Type: tf.Type.Name(),
+ Text: tf.Tag.Get("doc"),
+ Default: tf.Tag.Get("default"),
+ Rules: tf.Tag.Get("rules"),
+ Value: v.Field(i),
+ })
+ }
+ }
+ return entries
+}
+
+type docEntry struct {
+ Env string
+ Type string
+ Text string
+ Default string
+ Rules string
+ Value reflect.Value
+ Current []string
+}
diff --git a/config/doc_example.md b/config/doc_example.md
new file mode 100644
index 0000000..85df6e9
--- /dev/null
+++ b/config/doc_example.md
@@ -0,0 +1,11 @@
+# Configuration from Environment
+
+|Environment|Type|Default|Description & Rules|Values|
+|---|---|---|---|---|
+|API_LOGS_CLEANUP_DAYS|int64||Nr of days to keep before cleanup. Default 31.|N/A|
+|API_LOGS_INDEX_NAME|string||Name of index for api-logs (lowercase alpha-numerics with dashes, default: uafrica-v3-api-logs)|N/A|
+|API_LOGS_MAX_RESPONSE_SIZE|int64||Maximum length of response body stored. Defaults to 1024.|N/A|
+|API_LOGS_SEARCH_ADDRESSES|list of string||List of server addresses. Requires at least one, e.g. "https://localhost:9200" for local testing|[https://search-uafrica-v3-api-logs-fefgiypvmb3sg5wqohgsbqnzvq.af-south-1.es.amazonaws.com/]|
+|API_LOGS_SEARCH_PASSWORD|string||User password for HTTP basic auth. Defaults to admin for local testing.|[Aiz}a4ee]|
+|API_LOGS_SEARCH_USERNAME|string||User name for HTTP basic auth. Defaults to admin for local testing.|[uafrica]|
+|AUDIT_MAX_RESPONSE_SIZE|int64||Maximum length of response body stored. Defaults to 1024.|N/A|
\ No newline at end of file
diff --git a/config/struct.go b/config/struct.go
index f52c44b..84053ad 100644
--- a/config/struct.go
+++ b/config/struct.go
@@ -1,114 +1,48 @@
package config
import (
- "encoding/json"
- "os"
- "reflect"
"regexp"
- "sort"
- "strconv"
- "strings"
"gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/struct_utils"
)
-func Load(prefix string, configPtr interface{}) error {
+var (
+ prefixStructs = map[string]interface{}{}
+)
+
+func Load(prefix string, configStructPtr interface{}) error {
if !prefixRegex.MatchString(prefix) {
- return errors.Errorf("invalid config prefix \"%s\"", prefix)
- }
- if configPtr == nil {
- return errors.Errorf("Load(nil)")
- }
- t := reflect.TypeOf(configPtr)
- if t.Kind() != reflect.Ptr || t.Elem().Kind() != reflect.Struct {
- return errors.Errorf("%T is not &struct", configPtr)
- }
- v := reflect.ValueOf(configPtr)
- if err := load(prefix, t.Elem(), v.Elem()); err != nil {
- return errors.Wrapf(err, "failed to load config with prefix %s", prefix)
+ return errors.Errorf("config(%s) invalid prefix", prefix)
}
- if validator, ok := configPtr.(Validator); ok {
- if err := validator.Validate(); err != nil {
- return errors.Wrapf(err, "invalid config with prefix %s", prefix)
- }
- }
- return nil
-}
-type nameValue struct {
- name string
- value string
-}
+ //store before load in case it fails to be still part of docs
+ prefixStructs[prefix] = configStructPtr
-func load(prefix string, t reflect.Type, ptrValue reflect.Value) error {
- switch t.Kind() {
- case reflect.Struct:
- for i := 0; i < t.NumField(); i++ {
- f := t.Field(i)
- if err := load(prefix+"_"+strings.ToUpper(f.Name), f.Type, ptrValue.Field(i)); err != nil {
- return errors.Wrapf(err, "cannot load field")
- }
- }
+ //read os.Getenv()
+ nv := struct_utils.NamedValuesFromEnv(prefix)
- case reflect.Slice:
- //expect JSON list of values or just one value
- s := os.Getenv(prefix)
- if s != "" {
- if err := json.Unmarshal([]byte(s), ptrValue.Addr().Interface()); err != nil {
- return errors.Wrapf(err, "cannot read env %s=%s into %s", prefix, s, t.Name())
- }
- } else {
- //see if _1, _2, ... is used then construct a list with those values
- //(only applies to list of strings)
- values := map[string]string{}
- for _, x := range os.Environ() {
- parts := strings.SplitN(x, "=", 2)
- if len(parts) == 2 && strings.HasPrefix(parts[0], prefix+"_") {
- values[parts[0]] = parts[1]
- }
- }
- if len(values) > 0 {
- //add in sorted order
- list := []nameValue{}
- for n, v := range values {
- list = append(list, nameValue{name: n, value: v})
- }
- sort.Slice(list, func(i, j int) bool {
- return list[i].name < list[j].name
- })
- s := ""
- for _, nv := range list {
- if t.Elem().Kind() == reflect.String {
- s += ",\"" + nv.value + "\"" //quoted
- } else {
- s += "," + nv.value //unquoted
- }
- }
- s = "[" + s[1:] + "]"
- if err := json.Unmarshal([]byte(s), ptrValue.Addr().Interface()); err != nil {
- return errors.Wrapf(err, "cannot read env %s=%s into %s", prefix, s, t.Name())
- }
- }
- }
-
- case reflect.String:
- s := os.Getenv(prefix)
- if s != "" {
- ptrValue.Set(reflect.ValueOf(s))
- }
+ //parse into struct
+ unused, err := struct_utils.UnmarshalNamedValues(nv, configStructPtr)
+ if err != nil {
+ return errors.Wrapf(err, "config(%s) cannot load", prefix)
+ }
+ if len(unused) > 0 {
+ //we still use os.Getenv() elsewhere, so some variables may not be in the struct
+ //e.g. AUDIT_QUEUE_URL is read from queues/sqs/producer which match config(prefix="AUDIT")
+ //so we cannot yet fail here, which we should, because config setting not used is often
+ //a reason for errors, when we try to configure something, then it does not work, and
+ //we cannot figure out why, but the value we did set, might just be misspelled etc.
+ //so, for now - do not fail here, just report the unused values
+ logger.Warnf("Note unused env (might be used elsewhere) for config(%s): %+v", prefix, unused)
+ //return errors.Errorf("config(%s): unknown %+v", prefix, unused)
+ }
- case reflect.Int64:
- s := os.Getenv(prefix)
- if s != "" {
- i64, err := strconv.ParseInt(s, 10, 64)
- if err != nil {
- return errors.Errorf("%s=%s not integer value", prefix, s)
- }
- ptrValue.Set(reflect.ValueOf(i64))
+ if validator, ok := configStructPtr.(Validator); ok {
+ if err := validator.Validate(); err != nil {
+ return errors.Wrapf(err, "config(%s) is invalid", prefix)
}
-
- default:
- return errors.Errorf("cannot load config %s_... into %s kind %s", prefix, t.Name(), t.Kind())
}
return nil
}
diff --git a/config/struct_test.go b/config/struct_test.go
index 2f153ef..587f8bd 100644
--- a/config/struct_test.go
+++ b/config/struct_test.go
@@ -1,10 +1,15 @@
package config_test
import (
+ "encoding/json"
+ "fmt"
"os"
+ "strings"
"testing"
+ "time"
"gitlab.com/uafrica/go-utils/config"
+ "gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
)
@@ -12,59 +17,136 @@ func TestLoad(t *testing.T) {
logger.SetGlobalFormat(logger.NewConsole())
logger.SetGlobalLevel(logger.LevelDebug)
- os.Setenv("TEST_A", "123")
- os.Setenv("TEST_B", "abc")
- os.Setenv("TEST_C", "789")
+ //booleans
+ os.Setenv("TEST_VALUE_ENABLE_CACHE", "true")
+ os.Setenv("TEST_VALUE_DISABLE_LOG", "true")
+ os.Setenv("TEST_VALUE_ADMIN", "false")
- //list of value must be valid JSON, i.e. if list of int or list of string, it must be unquoted or quoted as expected by JSON:
- os.Setenv("TEST_L", "[1,2,3]")
- os.Setenv("TEST_M", "[\"7\", \"8\"]")
+ //integers
+ os.Setenv("TEST_VALUE_MAX_SIZE", "12")
- //list of string entries can also be defined with _1, _2, ... postfixes
- //the key value must be unique but has no significance apart from ordering,
- //so if you comment out _3, then _1, _2 and _4 will result in 3 entries in the list
- //as long as they are unique keys
- os.Setenv("TEST_N_1", "111")
- os.Setenv("TEST_N_2", "222")
- //os.Setenv("TEST_N_3", "333")
- os.Setenv("TEST_N_4", "444")
+ os.Setenv("TEST_VALUE_SEQ1", "[4,5,6]") //list in one value
- os.Setenv("TEST_P_1", "111")
- os.Setenv("TEST_P_2", "222")
- //os.Setenv("TEST_N_3", "333")
- os.Setenv("TEST_P_4", "444")
+ os.Setenv("TEST_VALUE_SEQ2_10", "10") //numbered list elements
+ os.Setenv("TEST_VALUE_SEQ2_20", "20")
+ os.Setenv("TEST_VALUE_SEQ2_4", "4")
+ os.Setenv("TEST_VALUE_SEQ2_15", "15")
+ os.Setenv("TEST_VALUE_SEQ2", "100")
+
+ os.Setenv("TEST_VALUE_CUTOFF", "2021-11-20T12:00:00+02:00")
+ os.Setenv("TEST_VALUE_HOLIDAYS", "[2021-03-21,2021-04-27,2021-05-01,2021-06-16,2021-08-09,2021-12-16,2021-12-25]")
c := Config{}
- if err := config.Load("TEST", &c); err != nil {
+ if err := config.Load("TEST_VALUE", &c); err != nil {
t.Fatalf("Cannot load config: %+v", err)
}
+ t.Logf("Loaded config: %+v", c)
- if c.A != "123" || c.B != "abc" || c.C != 789 {
- t.Fatalf("Loaded wrong values: %+v", c)
+ if !c.EnableCache || !c.DisableLog || c.Admin {
+ t.Fatalf("wrong bool values: %+v", c)
+ }
+ if c.MaxSize != 12 {
+ t.Fatalf("wrong nr values: %+v", c)
}
- if len(c.L) != 3 || c.L[0] != 1 || c.L[1] != 2 || c.L[2] != 3 {
- t.Fatalf("Loaded wrong values: %+v", c)
+ if len(c.Seq1) != 3 || c.Seq1[0] != 4 || c.Seq1[1] != 5 || c.Seq1[2] != 6 {
+ t.Fatalf("wrong seq1: %+v", c)
}
- if len(c.M) != 2 || c.M[0] != "7" || c.M[1] != "8" {
- t.Fatalf("Loaded wrong values for M: %+v", c.M)
+ if len(c.Seq2) != 5 || c.Seq2[0] != 100 || c.Seq2[1] != 4 || c.Seq2[2] != 10 || c.Seq2[3] != 15 || c.Seq2[4] != 20 {
+ t.Fatalf("wrong seq2: %+v", c)
}
- t.Logf("M=%+v", c.M)
- if len(c.N) != 3 || c.N[0] != "111" || c.N[1] != "222" || c.N[2] != "444" {
- t.Fatalf("Loaded wrong values for N: %+v", c.N)
+ if c.Cutoff.UTC().Format("2006-01-02 15:04:05") != "2021-11-20 10:00:00" {
+ t.Fatalf("wrong cutoff")
}
- t.Logf("N=%+v", c.N)
- if len(c.P) != 3 || c.P[0] != 111 || c.P[1] != 222 || c.P[2] != 444 {
- t.Fatalf("Loaded wrong values for P: %+v", c.N)
+ if len(c.Holidays) != 7 ||
+ c.Holidays[0].String() != "2021-03-21" ||
+ c.Holidays[1].String() != "2021-04-27" ||
+ c.Holidays[2].String() != "2021-05-01" ||
+ c.Holidays[3].String() != "2021-06-16" ||
+ c.Holidays[4].String() != "2021-08-09" ||
+ c.Holidays[5].String() != "2021-12-16" ||
+ c.Holidays[6].String() != "2021-12-25" {
+ t.Fatalf("wrong holidays")
+ }
+
+ {
+ t.Logf("config(TEST) = %+v", c)
+ e := json.NewEncoder(os.Stdout)
+ e.SetIndent("", " ")
+ e.Encode(c)
}
- t.Logf("P=%+v", c.P)
+
}
type Config struct {
- A string `json:"a"`
- B string `json:"b"`
- C int64 `json:"c"`
- L []int64 `json:"l"`
- M []string `json:"m"`
- N []string `json:"n"`
- P []int64 `json:"p"`
+ EnableCache bool `json:"enable_cache"`
+ DisableLog bool `json:"disable_log"`
+ Admin bool `json:"admin"`
+ MaxSize int64 `json:"max_size"`
+ Seq1 []int `json:"seq1"`
+ Seq2 []int64 `json:"seq2"`
+ Cutoff time.Time `json:"cutoff"`
+ Holidays []Date `json:"holidays"`
+}
+
+type Date struct {
+ Y, M, D int
+}
+
+func (d *Date) Scan(value []byte) error {
+ s := strings.Trim(string(value), "\"")
+ v, err := time.ParseInLocation("2006-01-02", s, time.Now().Location())
+ if err != nil {
+ return errors.Errorf("%s is not CCYY-MM-DD", s)
+ }
+ d.Y = v.Year()
+ d.M = int(v.Month())
+ d.D = v.Day()
+ return nil
+}
+
+func (d *Date) UnmarshalJSON(value []byte) error {
+ return d.Scan(value)
+}
+
+func (d Date) String() string {
+ return fmt.Sprintf("%04d-%02d-%02d", d.Y, d.M, d.D)
+}
+
+func (d Date) MarshalJSON() ([]byte, error) {
+ return []byte("\"" + d.String() + "\""), nil
+}
+
+type SearchConfig struct {
+ Addresses []string `json:"addresses"`
+}
+
+type LogConfig struct {
+ SearchConfig `json:"search"`
+ Search2 SearchConfig `json:"search2"`
+ IndexName string `json:"index_name"`
+}
+
+func TestLogConfig(t *testing.T) {
+ logger.SetGlobalFormat(logger.NewConsole())
+ logger.SetGlobalLevel(logger.LevelDebug)
+ os.Setenv("LOG_INDEX_NAME", "abc")
+ os.Setenv("LOG_SEARCH_ADDRESSES", "[A,B,C]")
+ os.Setenv("LOG_SEARCH2_ADDRESSES", "[D,E,F]")
+ os.Setenv("LOG_OTHER", "1")
+ os.Setenv("LOG_SEARCH_OTHER", "2")
+ c := LogConfig{}
+ err := config.Load("LOG", &c)
+ if err != nil {
+ t.Fatalf("Failed: %+v", err)
+ }
+ t.Logf("Loaded: %+v", c)
+ if c.IndexName != "abc" {
+ t.Fatalf("wrong index_name:%s", c.IndexName)
+ }
+ if len(c.Addresses) != 3 || c.Addresses[0] != "A" || c.Addresses[1] != "B" || c.Addresses[2] != "C" {
+ t.Fatalf("wrong addresses:%+v", c.Addresses)
+ }
+ if len(c.Search2.Addresses) != 3 || c.Search2.Addresses[0] != "D" || c.Search2.Addresses[1] != "E" || c.Search2.Addresses[2] != "F" {
+ t.Fatalf("wrong search2 addresses:%+v", c.Search2.Addresses)
+ }
}
diff --git a/struct_utils/named_values_to_struct.go b/struct_utils/named_values_to_struct.go
new file mode 100644
index 0000000..2f118e4
--- /dev/null
+++ b/struct_utils/named_values_to_struct.go
@@ -0,0 +1,294 @@
+package struct_utils
+
+import (
+ "encoding/csv"
+ "encoding/json"
+ "os"
+ "reflect"
+ "sort"
+ "strconv"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+//Purpose:
+// Make a list of named values from the env for parsing into a struct
+//
+//Parameters:
+// prefix should be uppercase (by convention) env prefix like "MY_LIB_CONFIG", without trailing "_"
+//
+//Result:
+// named values that can be passed into UnmarshalNamedValues()
+//
+//All env starting with "<prefix>_" will be copied without "<prefix>_"
+//Examples with prefix="MY_LIB_CONFIG":
+// MY_LIB_CONFIG_MAX_SIZE="6" -> {"MAX_SIZE":["6"]} one value of "6"
+// MY_LIB_CONFIG_NAMES ="A,B,C" -> {"NAMES":["A,B,C"]} one value of "A,B,C"
+// MY_LIB_CONFIG_NRS ="1,2,3" -> {"NRS":["1,2,3"]} one value of "1,2,3" (all env values are string, later parsed into int based on struct field type)
+// MY_LIB_CONFIG_CODES ="[1,2,3]"" -> {"CODES":["1","2","3"]} 3 values of "1", "2" and "3" because of outer [...], env values are string
+//
+// MY_LIB_CONFIG_CODES_1=5
+// MY_LIB_CONFIG_CODES_5=7
+// MY_LIB_CONFIG_CODES_2=10 -> {"CODES":["5","10","7"]} 3 values ordered on suffixes "_1", "_5", "_2" moving 10 before 7
+//
+// MY_LIB_CONFIG_ADDRS=["55 Crescent, Town", "12 Big Street, City"] -> 2 values including commas because of quoted CSV
+func NamedValuesFromEnv(prefix string) map[string][]string {
+ result := map[string][]string{}
+ prefix += "_"
+ for _, env := range os.Environ() {
+ if strings.HasPrefix(env, prefix) {
+ parts := strings.SplitN(env[len(prefix):], "=", 2)
+ if len(parts) == 2 {
+ //name = parts[0], value = parts[1]
+ value := parts[1]
+ result[strings.ToLower(parts[0])] = []string{value}
+
+ //split only if valid CSV between [...]
+ if value[0] == '[' && value[len(value)-1] == ']' {
+ csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
+ csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
+ if csvErr == nil {
+ result[strings.ToLower(parts[0])] = csvValues
+ }
+ }
+ }
+ }
+ }
+
+ //merge multiple <name>_#=<value> into single lists called <name>
+ namesToDelete := []string{}
+ merged := map[string][]nrWithValues{}
+ for name, values := range result {
+ delimIndex := strings.LastIndex(name, "_")
+ if delimIndex > 0 {
+ nr := name[delimIndex+1:]
+ if i64, err := strconv.ParseInt(nr, 10, 64); err == nil {
+ nameWithoutNr := name[:delimIndex]
+ if _, ok := merged[nameWithoutNr]; !ok {
+ merged[nameWithoutNr] = []nrWithValues{}
+ }
+ merged[nameWithoutNr] = append(merged[nameWithoutNr], nrWithValues{nr: i64, values: values})
+ namesToDelete = append(namesToDelete, name)
+ }
+ }
+ }
+
+ //delete merged values
+ for _, name := range namesToDelete {
+ delete(result, name)
+ }
+
+ //sort and set the merged names with single list of values
+ for nameWithoutNr, nrsWithValues := range merged {
+ if values, ok := result[nameWithoutNr]; ok {
+ nrsWithValues = append(nrsWithValues, nrWithValues{nr: 0, values: values}) //if also defined without _#
+ }
+ sort.Slice(nrsWithValues, func(i, j int) bool { return nrsWithValues[i].nr < nrsWithValues[j].nr })
+
+ list := []string{}
+ for _, nrWithValues := range nrsWithValues {
+ list = append(list, nrWithValues.values...)
+ }
+ result[nameWithoutNr] = list
+ }
+ return result
+}
+
+type nrWithValues struct {
+ nr int64
+ values []string
+}
+
+//converts query string params to named values that can be parsed into a struct
+//it support both single/multi-value params, depending how you get them from your HTTP library
+// (e.g. AWS API Gateway Context returns both but default golang net/http returns only params)
+func NamedValuesFromURL(params map[string]string, multiValueParams map[string][]string) map[string][]string {
+ result := map[string][]string{}
+ for n, v := range params {
+ result[n] = []string{v}
+ }
+ for n, mv := range multiValueParams {
+ if list, ok := result[n]; !ok {
+ result[n] = mv
+ } else {
+ //do not add duplicates - seems like AWS put same value in both single and multivalue params
+ for _, v := range mv {
+ found := false
+ for _, existingValue := range list {
+ if v == existingValue {
+ found = true
+ }
+ }
+ if !found {
+ list = append(list, v)
+ }
+ }
+ result[n] = list
+ }
+ }
+
+ for name, values := range result {
+ splitValues := []string{}
+ for _, value := range values {
+ //split only if valid CSV between [...]
+ if value[0] == '[' && value[len(value)-1] == ']' {
+ csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
+ csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
+ if csvErr == nil {
+ splitValues = append(splitValues, csvValues...)
+ } else {
+ splitValues = append(splitValues, value) //cannot split this "[...]" value
+ }
+ } else {
+ splitValues = append(splitValues, value) //not a "[...]" value
+ }
+ }
+ result[name] = splitValues
+ }
+ return result
+}
+
+// Purpose:
+// UnmarshalNamedValues() parses a set of named values into a struct using json tag matching
+// Unlike json.Unmarshal(), it takes care of converting quoted "true" -> true, "1" -> int(1) etc...
+//
+// Typically used to parse environment or URL params into a struct
+// because normal json.Unmarshal() will fail to parse "2" into an integer etc
+//
+// By convention, the names should be lowercase to match json tag with "_" delimeters
+// And also use "_" for nested sub-struct names
+// named value "a_b_c_d":5 would be stored in
+// field with json tag "a_b_c_d" or
+// field with json tag "a_b" which is a struct with a json tagged field "c_d" etc...
+//
+// Parameters:
+// namedValues is name-value pairs, typical from URL params or OS environment
+// see construction functions for this:
+// NamedValuesFromEnv()
+// NamedValuesFromURL()
+//
+// structPtr must be ptr to a struct variable
+// undefined values will not be changed, so you can call this multiple times on the
+// same struct to amend a few values, leaving the rest and default values unchanged
+//
+// Return:
+// unused values
+// nil or error if some values could not be used
+//
+// If all values must be used, check len(unusedValues) when err==nil
+//
+func UnmarshalNamedValues(namedValues map[string][]string, structPtr interface{}) (unusedValues map[string][]string, err error) {
+ if structPtr == nil {
+ return nil, errors.Errorf("cannot unmarshal into nil")
+ }
+ structPtrType := reflect.TypeOf(structPtr)
+ if structPtrType.Kind() != reflect.Ptr || structPtrType.Elem().Kind() != reflect.Struct {
+ return nil, errors.Errorf("%T is not &struct", structPtr)
+ }
+ structType := structPtrType.Elem()
+ structPtrValue := reflect.ValueOf(structPtr)
+
+ if usedNameList, err := unmarshalNamedValuesIntoStructPtr("", namedValues, structType, structPtrValue); err != nil {
+ return namedValues, err
+ } else {
+ for _, usedName := range usedNameList {
+ delete(namedValues, usedName)
+ }
+ }
+ return namedValues, nil
+}
+
+func unmarshalNamedValuesIntoStructPtr(prefix string, namedValues map[string][]string, structType reflect.Type, structPtrValue reflect.Value) (usedNameList []string, err error) {
+ usedNameList = []string{}
+ for i := 0; i < structType.NumField(); i++ {
+ structTypeField := structType.Field(i)
+ fieldName := (strings.SplitN(structTypeField.Tag.Get("json"), ",", 2))[0]
+ if fieldName == "-" {
+ continue //skip fields excluded from JSON
+ }
+ if prefix != "" {
+ if fieldName != "" {
+ fieldName = prefix + "_" + fieldName
+ } else {
+ fieldName = prefix
+ }
+ }
+
+ //recurse into anonymous sub-structs
+ if structTypeField.Type.Kind() == reflect.Struct {
+ if nameList, err := unmarshalNamedValuesIntoStructPtr(fieldName, namedValues, structTypeField.Type, structPtrValue.Elem().Field(i).Addr()); err != nil {
+ return nil, errors.Wrapf(err, "failed on %s.%s", structType.Name(), fieldName)
+ } else {
+ usedNameList = append(usedNameList, nameList...)
+ }
+ continue
+ }
+
+ fieldValues, ok := namedValues[fieldName]
+ if !ok {
+ continue //skip undefined fields
+ }
+ usedNameList = append(usedNameList, fieldName)
+ delete(namedValues, fieldName)
+ if len(fieldValues) == 0 {
+ continue //field has no value specified in URL, do not remove values not defined (cannot clear defined struct fields with named values)
+ }
+ structPtrFieldValue := structPtrValue.Elem().Field(i)
+ if structPtrFieldValue.Kind() == reflect.Ptr {
+ //this is a ptr, allocate a new value and set it
+ //then we can dereference to set it below
+ structPtrFieldValue.Set(reflect.New(structPtrFieldValue.Type().Elem()))
+ structPtrFieldValue = structPtrFieldValue.Elem()
+ }
+
+ //param is defined >=1 times in URL
+ if structTypeField.Type.Kind() == reflect.Slice {
+ //this param struct field is a slice, iterate over all specified values
+ for i, fieldValue := range fieldValues {
+ parsedValue, parseErr := unmarshalValue(fieldValue, structTypeField.Type.Elem())
+ if parseErr != nil {
+ err = errors.Wrapf(parseErr, "invalid %s[%d]", fieldName, i)
+ return
+ }
+ structPtrFieldValue.Set(reflect.Append(structPtrFieldValue, parsedValue))
+
+ //todo: sorting of list using value names as applicable
+ }
+ } else {
+ //field is not a slice, expecting only a single value
+ if len(fieldValues) > 1 {
+ err = errors.Errorf("%s cannot store multiple value (%d found: %+v)", fieldName, len(fieldValues), fieldValues)
+ return
+ }
+ parsedValue, parseErr := unmarshalValue(fieldValues[0], structPtrFieldValue.Type())
+ if parseErr != nil {
+ err = errors.Wrapf(parseErr, "invalid %s", fieldName)
+ return
+ }
+ structPtrFieldValue.Set(parsedValue)
+ }
+ } //for each param struct field
+ return usedNameList, nil
+}
+
+func unmarshalValue(v interface{}, t reflect.Type) (reflect.Value, error) {
+ newValuePtr := reflect.New(t)
+ if reflect.ValueOf(v).Type().AssignableTo(t) {
+ newValuePtr.Elem().Set(reflect.ValueOf(v)) //can assign as is
+ } else {
+ //needs conversion
+ s, ok := v.(string)
+ if !ok {
+ jsonValue, _ := json.Marshal(v)
+ s = string(jsonValue)
+ }
+ //is string value, unmarshal as quoted or unquoted JSON value
+ if err := json.Unmarshal([]byte("\""+s+"\""), newValuePtr.Interface()); err != nil {
+ if err := json.Unmarshal([]byte(s), newValuePtr.Interface()); err != nil {
+ return newValuePtr.Elem(), errors.Wrapf(err, "invalid \"%s\"", s)
+ }
+ }
+ }
+ return newValuePtr.Elem(), nil
+}
diff --git a/struct_utils/named_values_to_struct_test.go b/struct_utils/named_values_to_struct_test.go
new file mode 100644
index 0000000..533cb16
--- /dev/null
+++ b/struct_utils/named_values_to_struct_test.go
@@ -0,0 +1,210 @@
+package struct_utils_test
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/struct_utils"
+)
+
+func TestEnv(t *testing.T) {
+ logger.SetGlobalFormat(logger.NewConsole())
+ logger.SetGlobalLevel(logger.LevelDebug)
+ //booleans
+ os.Setenv("TEST_VALUE_ENABLE_CACHE", "true")
+ os.Setenv("TEST_VALUE_DISABLE_LOG", "true")
+ os.Setenv("TEST_VALUE_ADMIN", "false")
+
+ //integers
+ os.Setenv("TEST_VALUE_MAX_SIZE", "12")
+
+ os.Setenv("TEST_VALUE_SEQ1", "[4,5,6]") //list in one value
+
+ os.Setenv("TEST_VALUE_SEQ2_10", "10") //numbered list elements
+ os.Setenv("TEST_VALUE_SEQ2_20", "20")
+ os.Setenv("TEST_VALUE_SEQ2_4", "4")
+ os.Setenv("TEST_VALUE_SEQ2_15", "15")
+ os.Setenv("TEST_VALUE_SEQ2", "100")
+
+ os.Setenv("TEST_VALUE_CUTOFF", "2021-11-20T12:00:00+02:00")
+ os.Setenv("TEST_VALUE_HOLIDAYS", "[2021-03-21,2021-04-27,2021-05-01,2021-06-16,2021-08-09,2021-12-16,2021-12-25]")
+
+ //=====[ TEST THIS FUNCTION ]=====
+ nv := struct_utils.NamedValuesFromEnv("TEST_VALUE")
+
+ testNamedValues(t, nv)
+}
+
+func TestURL1(t *testing.T) {
+ logger.SetGlobalFormat(logger.NewConsole())
+ logger.SetGlobalLevel(logger.LevelDebug)
+
+ queryParams := map[string]string{
+ "enable_cache": "true",
+ "disable_log": "true",
+ "admin": "false",
+ "max_size": "12",
+ "seq1": "[4,5,6]",
+ "seq2": "[100,4,10,15,20]", //url does not support _# numbering of params in a list, only [csv] or multi-value with seq2=100&seq2=4&... testing in TestURL2()
+ "cutoff": "2021-11-20T12:00:00+02:00",
+ "holidays": "[2021-03-21,2021-04-27,2021-05-01,2021-06-16,2021-08-09,2021-12-16,2021-12-25]",
+ }
+
+ //=====[ TEST THIS FUNCTION ]=====
+ nv := struct_utils.NamedValuesFromURL(queryParams, nil)
+ testNamedValues(t, nv)
+}
+
+func TestURL2(t *testing.T) {
+ logger.SetGlobalFormat(logger.NewConsole())
+ logger.SetGlobalLevel(logger.LevelDebug)
+
+ queryParams := map[string]string{
+ "disable_log": "true",
+ "admin": "false",
+ "max_size": "12",
+ "seq1": "4",
+ "cutoff": "2021-11-20T12:00:00+02:00",
+ }
+ multiValueParams := map[string][]string{
+ "enable_cache": {"true"},
+ "seq1": {"5", "6"}, //merged with above "4"
+ "seq2": {"100", "4", "10", "15", "20"},
+ "holidays": {"2021-03-21", "2021-04-27", "2021-05-01", "2021-06-16", "2021-08-09", "2021-12-16", "2021-12-25"},
+ }
+
+ //=====[ TEST THIS FUNCTION ]=====
+ nv := struct_utils.NamedValuesFromURL(queryParams, multiValueParams)
+ testNamedValues(t, nv)
+}
+
+func TestURL3(t *testing.T) {
+ urlString := "/test?admin=false&cutoff=2021-11-20T12%3A00%3A00%2B02%3A00&disable_log=true&enable_cache=true&holidays=2021-03-21&holidays=2021-04-27&holidays=2021-05-01&holidays=2021-06-16&holidays=2021-08-09&holidays=2021-12-16&holidays=2021-12-25&max_size=12&seq1=4&seq1=5&seq1=6&seq2=100&seq2=4&seq2=10&seq2=15&seq2=20"
+ u, err := url.Parse(urlString)
+ if err != nil {
+ t.Fatalf("cannot parse URL")
+ }
+ nv := struct_utils.NamedValuesFromURL(nil, u.Query())
+ testNamedValues(t, nv)
+}
+
+func testNamedValues(t *testing.T, nv map[string][]string) {
+ //assets expected values
+ exp := map[string][]string{
+ "enable_cache": {"true"},
+ "disable_log": {"true"},
+ "admin": {"false"},
+ "max_size": {"12"},
+ "seq1": {"4", "5", "6"},
+ "seq2": {"100", "4", "10", "15", "20"}, //order is important
+ "cutoff": {"2021-11-20T12:00:00+02:00"},
+ "holidays": {"2021-03-21", "2021-04-27", "2021-05-01", "2021-06-16", "2021-08-09", "2021-12-16", "2021-12-25"},
+ }
+ if len(nv) != len(exp) {
+ t.Fatalf("len(nv)=%d != len(exp)=%d: %+v", len(nv), len(exp), nv)
+ }
+
+ for name, expValues := range exp {
+ if envValues, ok := nv[name]; !ok {
+ t.Fatalf("%s not set in %+v", name, nv)
+ } else {
+ if len(expValues) != len(envValues) {
+ t.Fatalf("%s has %d != %d values (%v != %v)", name, len(envValues), len(expValues), envValues, expValues)
+ }
+ for i, v := range envValues {
+ if v != expValues[i] {
+ t.Fatalf("%s[%d] = \"%s\" != \"%s\"", name, i, v, expValues[i])
+ }
+ }
+ }
+ }
+
+ //=====[ PARSE INTO STRUCT ]==========
+ c := Config{}
+ unused, err := struct_utils.UnmarshalNamedValues(nv, &c)
+ if err != nil {
+ t.Fatalf("failed: %+v", err)
+ }
+ if len(unused) != 0 {
+ t.Fatalf("unused: %+v", unused)
+ }
+ t.Logf("parsed struct: %+v", c)
+
+ {
+ e := json.NewEncoder(os.Stdout)
+ e.SetIndent("", "\t")
+ e.Encode(c)
+ }
+
+ if !c.EnableCache || !c.DisableLog || c.Admin {
+ t.Fatalf("wrong bool values: %+v", c)
+ }
+ if c.MaxSize != 12 {
+ t.Fatalf("wrong nr values: %+v", c)
+ }
+ if len(c.Seq1) != 3 || c.Seq1[0] != 4 || c.Seq1[1] != 5 || c.Seq1[2] != 6 {
+ t.Fatalf("wrong seq1: %+v", c)
+ }
+ if len(c.Seq2) != 5 || c.Seq2[0] != 100 || c.Seq2[1] != 4 || c.Seq2[2] != 10 || c.Seq2[3] != 15 || c.Seq2[4] != 20 {
+ t.Fatalf("wrong seq2: %+v", c)
+ }
+ if c.Cutoff.UTC().Format("2006-01-02 15:04:05") != "2021-11-20 10:00:00" {
+ t.Fatalf("wrong cutoff")
+ }
+ if len(c.Holidays) != 7 ||
+ c.Holidays[0].String() != "2021-03-21" ||
+ c.Holidays[1].String() != "2021-04-27" ||
+ c.Holidays[2].String() != "2021-05-01" ||
+ c.Holidays[3].String() != "2021-06-16" ||
+ c.Holidays[4].String() != "2021-08-09" ||
+ c.Holidays[5].String() != "2021-12-16" ||
+ c.Holidays[6].String() != "2021-12-25" {
+ t.Fatalf("wrong holidays")
+ }
+}
+
+type Config struct {
+ EnableCache bool `json:"enable_cache"`
+ DisableLog bool `json:"disable_log"`
+ Admin bool `json:"admin"`
+ MaxSize int64 `json:"max_size"`
+ Seq1 []int `json:"seq1"`
+ Seq2 []int64 `json:"seq2"`
+ Cutoff time.Time `json:"cutoff"`
+ Holidays []Date `json:"holidays"`
+}
+
+type Date struct {
+ Y, M, D int
+}
+
+func (d *Date) Scan(value []byte) error {
+ s := strings.Trim(string(value), "\"")
+ v, err := time.ParseInLocation("2006-01-02", s, time.Now().Location())
+ if err != nil {
+ return errors.Errorf("%s is not CCYY-MM-DD", s)
+ }
+ d.Y = v.Year()
+ d.M = int(v.Month())
+ d.D = v.Day()
+ return nil
+}
+
+func (d *Date) UnmarshalJSON(value []byte) error {
+ return d.Scan(value)
+}
+
+func (d Date) String() string {
+ return fmt.Sprintf("%04d-%02d-%02d", d.Y, d.M, d.D)
+}
+
+func (d Date) MarshalJSON() ([]byte, error) {
+ return []byte("\"" + d.String() + "\""), nil
+}
--
GitLab