package struct_utils import ( "database/sql" "encoding/csv" "encoding/json" "reflect" "sort" "strconv" "strings" "gitlab.com/uafrica/go-utils/errors" "gitlab.com/uafrica/go-utils/logs" "gitlab.com/uafrica/go-utils/string_utils" ) //Purpose: // Make a list of named values from the env for parsing into a struct // //Parameters: // prefix should be uppercase (by convention) env prefix like "MY_LIB_CONFIG", without trailing "_" // //Result: // named values that can be passed into UnmarshalNamedValues() // //All env starting with "<prefix>_" will be copied without "<prefix>_" //Examples with prefix="MY_LIB_CONFIG": // MY_LIB_CONFIG_MAX_SIZE="6" -> {"MAX_SIZE":["6"]} one value of "6" // MY_LIB_CONFIG_NAMES ="A,B,C" -> {"NAMES":["A,B,C"]} one value of "A,B,C" // MY_LIB_CONFIG_NRS ="1,2,3" -> {"NRS":["1,2,3"]} one value of "1,2,3" (all env values are string, later parsed into int based on struct field type) // MY_LIB_CONFIG_CODES ="[1,2,3]"" -> {"CODES":["1","2","3"]} 3 values of "1", "2" and "3" because of outer [...], env values are string // // MY_LIB_CONFIG_CODES_1=5 // MY_LIB_CONFIG_CODES_5=7 // MY_LIB_CONFIG_CODES_2=10 -> {"CODES":["5","10","7"]} 3 values ordered on suffixes "_1", "_5", "_2" moving 10 before 7 // // MY_LIB_CONFIG_ADDRS=["55 Crescent, Town", "12 Big Street, City"] -> 2 values including commas because of quoted CSV func NamedValuesFromEnv(prefix string) map[string][]string { return NamedValuesFromReader(prefix, string_utils.EnvironmentKeyReader()) } func NamedValuesFromReader(prefix string, reader string_utils.KeyReader) map[string][]string { if reader == nil { return nil } result := map[string][]string{} prefix += "_" for _, key := range reader.Keys(prefix) { value, ok := reader.GetString(key) key = key[len(prefix):] if !ok { logs.Warn("Key(%s) undefined", key) continue } result[strings.ToLower(key)] = []string{value} //split only if valid CSV between [...] if value[0] == '[' && value[len(value)-1] == ']' { csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1])) csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...] if csvErr == nil { result[strings.ToLower(key)] = csvValues } } } //merge multiple <name>_#=<value> into single lists called <name> namesToDelete := []string{} merged := map[string][]nrWithValues{} for name, values := range result { delimIndex := strings.LastIndex(name, "_") if delimIndex > 0 { nr := name[delimIndex+1:] if i64, err := strconv.ParseInt(nr, 10, 64); err == nil { nameWithoutNr := name[:delimIndex] if _, ok := merged[nameWithoutNr]; !ok { merged[nameWithoutNr] = []nrWithValues{} } merged[nameWithoutNr] = append(merged[nameWithoutNr], nrWithValues{nr: i64, values: values}) namesToDelete = append(namesToDelete, name) } } } //delete merged values for _, name := range namesToDelete { delete(result, name) } //sort and set the merged names with single list of values for nameWithoutNr, nrsWithValues := range merged { if values, ok := result[nameWithoutNr]; ok { nrsWithValues = append(nrsWithValues, nrWithValues{nr: 0, values: values}) //if also defined without _# } sort.Slice(nrsWithValues, func(i, j int) bool { return nrsWithValues[i].nr < nrsWithValues[j].nr }) list := []string{} for _, nrWithValues := range nrsWithValues { list = append(list, nrWithValues.values...) } result[nameWithoutNr] = list } return result } type nrWithValues struct { nr int64 values []string } //converts query string params to named values that can be parsed into a struct //it support both single/multi-value params, depending how you get them from your HTTP library // (e.g. AWS API Gateway Context returns both but default golang net/http returns only params) func NamedValuesFromURL(params map[string]string, multiValueParams map[string][]string) map[string][]string { result := map[string][]string{} for n, v := range params { result[n] = []string{v} } for n, mv := range multiValueParams { if list, ok := result[n]; !ok { result[n] = mv } else { //do not add duplicates - seems like AWS put same value in both single and multivalue params for _, v := range mv { found := false for _, existingValue := range list { if v == existingValue { found = true } } if !found { list = append(list, v) } } result[n] = list } } for name, values := range result { splitValues := []string{} for _, value := range values { //split only if valid CSV between [...] if value == "" { continue } if value[0] == '[' && value[len(value)-1] == ']' { csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1])) csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...] if csvErr == nil { splitValues = append(splitValues, csvValues...) } else { splitValues = append(splitValues, value) //cannot split this "[...]" value } } else { splitValues = append(splitValues, value) //not a "[...]" value } } result[name] = splitValues } return result } // Purpose: // UnmarshalNamedValues() parses a set of named values into a struct using json tag matching // Unlike json.Unmarshal(), it takes care of converting quoted "true" -> true, "1" -> int(1) etc... // // Typically used to parse environment or URL params into a struct // because normal json.Unmarshal() will fail to parse "2" into an integer etc // // By convention, the names should be lowercase to match json tag with "_" delimeters // And also use "_" for nested sub-struct names // named value "a_b_c_d":5 would be stored in // field with json tag "a_b_c_d" or // field with json tag "a_b" which is a struct with a json tagged field "c_d" etc... // // Parameters: // namedValues is name-value pairs, typical from URL params or OS environment // see construction functions for this: // NamedValuesFromEnv() // NamedValuesFromURL() // // structPtr must be ptr to a struct variable // undefined values will not be changed, so you can call this multiple times on the // same struct to amend a few values, leaving the rest and default values unchanged // // Return: // unused values // nil or error if some values could not be used // // If all values must be used, check len(unusedValues) when err==nil // func UnmarshalNamedValues(namedValues map[string][]string, structPtr interface{}) (unusedValues map[string][]string, err error) { if structPtr == nil { return nil, errors.Errorf("cannot unmarshal into nil") } structPtrType := reflect.TypeOf(structPtr) if structPtrType.Kind() != reflect.Ptr || structPtrType.Elem().Kind() != reflect.Struct { return nil, errors.Errorf("%T is not &struct", structPtr) } structType := structPtrType.Elem() structPtrValue := reflect.ValueOf(structPtr) if usedNameList, err := unmarshalNamedValuesIntoStructPtr("", namedValues, structType, structPtrValue); err != nil { return namedValues, err } else { for _, usedName := range usedNameList { delete(namedValues, usedName) } } return namedValues, nil } func unmarshalNamedValuesIntoStructPtr(prefix string, namedValues map[string][]string, structType reflect.Type, structPtrValue reflect.Value) (usedNameList []string, err error) { usedNameList = []string{} for i := 0; i < structType.NumField(); i++ { structTypeField := structType.Field(i) fieldName := (strings.SplitN(structTypeField.Tag.Get("json"), ",", 2))[0] if fieldName == "-" { continue //skip fields excluded from JSON } if prefix != "" { if fieldName != "" { fieldName = prefix + "_" + fieldName } else { fieldName = prefix } } //recurse into anonymous sub-structs if structTypeField.Type.Kind() == reflect.Struct { if nameList, err := unmarshalNamedValuesIntoStructPtr(fieldName, namedValues, structTypeField.Type, structPtrValue.Elem().Field(i).Addr()); err != nil { return nil, errors.Wrapf(err, "failed on %s.%s", structType.Name(), fieldName) } else { usedNameList = append(usedNameList, nameList...) } continue } fieldValues, ok := namedValues[fieldName] if !ok { continue //skip undefined fields } usedNameList = append(usedNameList, fieldName) delete(namedValues, fieldName) if len(fieldValues) == 0 { continue //field has no value specified in URL, do not remove values not defined (cannot clear defined struct fields with named values) } structPtrFieldValue := structPtrValue.Elem().Field(i) if structPtrFieldValue.Kind() == reflect.Ptr { //this is a ptr, allocate a new value and set it //then we can dereference to set it below structPtrFieldValue.Set(reflect.New(structPtrFieldValue.Type().Elem())) structPtrFieldValue = structPtrFieldValue.Elem() } //param is defined >=1 times in URL if structTypeField.Type.Kind() == reflect.Slice { //this param struct field is a slice, iterate over all specified values for i, fieldValue := range fieldValues { parsedValue, parseErr := unmarshalValue(fieldValue, structTypeField.Type.Elem()) if parseErr != nil { err = errors.Wrapf(parseErr, "invalid %s[%d]", fieldName, i) return } structPtrFieldValue.Set(reflect.Append(structPtrFieldValue, parsedValue)) //todo: sorting of list using value names as applicable } } else { //field is not a slice, expecting only a single value if len(fieldValues) > 1 { err = errors.Errorf("%s cannot store multiple value (%d found: %+v)", fieldName, len(fieldValues), fieldValues) return } parsedValue, parseErr := unmarshalValue(fieldValues[0], structPtrFieldValue.Type()) if parseErr != nil { err = errors.Wrapf(parseErr, "invalid %s", fieldName) return } structPtrFieldValue.Set(parsedValue) } } //for each param struct field return usedNameList, nil } func unmarshalValue(v interface{}, t reflect.Type) (reflect.Value, error) { newValuePtr := reflect.New(t) if reflect.ValueOf(v).Type().AssignableTo(t) { newValuePtr.Elem().Set(reflect.ValueOf(v)) //can assign as is return newValuePtr.Elem(), nil } //needs conversion s, ok := v.(string) if !ok { jsonValue, _ := json.Marshal(v) s = string(jsonValue) } //now we have string value if valueScanner, ok := newValuePtr.Interface().(sql.Scanner); ok { //if has scanner - prefer that over json unmarshal //because we do not know if json expects quoted/unquoted for this type //and if we try quoted, it fail, then try unquoted, we have two different //errors one one of them will be kind of meaning nothing, but which? //scanner should always take the value as typed, so let's try that first if err := valueScanner.Scan(s); err == nil { return newValuePtr.Elem(), nil } } //try JSON unmarshal as is else with quotes if err := json.Unmarshal([]byte(s), newValuePtr.Interface()); err == nil { return newValuePtr.Elem(), nil } if err := json.Unmarshal([]byte("\""+s+"\""), newValuePtr.Interface()); err != nil { return newValuePtr.Elem(), errors.Wrapf(err, "invalid \"%s\"", s) } return newValuePtr.Elem(), nil }