named_values_to_struct.go

package struct_utils

import (
	"encoding/csv"
	"encoding/json"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"gitlab.com/uafrica/go-utils/errors"
	"gitlab.com/uafrica/go-utils/logger"
	"gitlab.com/uafrica/go-utils/string_utils"
)

//Purpose:
//	Make a list of named values from the env for parsing into a struct
//
//Parameters:
//	prefix should be uppercase (by convention) env prefix like "MY_LIB_CONFIG", without trailing "_"
//
//Result:
//	named values that can be passed into UnmarshalNamedValues()
//
//All env starting with "<prefix>_" will be copied without "<prefix>_"
//Examples with prefix="MY_LIB_CONFIG":
// MY_LIB_CONFIG_MAX_SIZE="6"        -> {"MAX_SIZE":["6"]}			one value of "6"
// MY_LIB_CONFIG_NAMES   ="A,B,C"    -> {"NAMES":["A,B,C"]}         one value of "A,B,C"
// MY_LIB_CONFIG_NRS     ="1,2,3"    -> {"NRS":["1,2,3"]}           one value of "1,2,3" (all env values are string, later parsed into int based on struct field type)
// MY_LIB_CONFIG_CODES   ="[1,2,3]"" -> {"CODES":["1","2","3"]}     3 values of "1", "2" and "3" because of outer [...], env values are string
//
// MY_LIB_CONFIG_CODES_1=5
// MY_LIB_CONFIG_CODES_5=7
// MY_LIB_CONFIG_CODES_2=10          -> {"CODES":["5","10","7"]}	3 values ordered on suffixes "_1", "_5", "_2" moving 10 before 7
//
// MY_LIB_CONFIG_ADDRS=["55 Crescent, Town", "12 Big Street, City"] -> 2 values including commas because of quoted CSV
func NamedValuesFromEnv(prefix string) map[string][]string {
	return NamedValuesFromReader(prefix, string_utils.EnvironmentKeyReader())
}

func NamedValuesFromReader(prefix string, reader string_utils.KeyReader) map[string][]string {
	if reader == nil {
		return nil
	}
	result := map[string][]string{}
	prefix += "_"
	for _, key := range reader.Keys(prefix) {
		value, ok := reader.GetString(key)
		key = key[len(prefix):]
		if !ok {
			logger.Debugf("Key(%s) undefined", key)
			continue
		}
		logger.Debugf("key(%s)=\"%s\"", key, value)
		result[strings.ToLower(key)] = []string{value}

		//split only if valid CSV between [...]
		if value[0] == '[' && value[len(value)-1] == ']' {
			csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
			csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
			if csvErr == nil {
				result[strings.ToLower(key)] = csvValues
			}
		}
	}

	//merge multiple <name>_#=<value> into single lists called <name>
	namesToDelete := []string{}
	merged := map[string][]nrWithValues{}
	for name, values := range result {
		delimIndex := strings.LastIndex(name, "_")
		if delimIndex > 0 {
			nr := name[delimIndex+1:]
			if i64, err := strconv.ParseInt(nr, 10, 64); err == nil {
				nameWithoutNr := name[:delimIndex]
				if _, ok := merged[nameWithoutNr]; !ok {
					merged[nameWithoutNr] = []nrWithValues{}
				}
				merged[nameWithoutNr] = append(merged[nameWithoutNr], nrWithValues{nr: i64, values: values})
				namesToDelete = append(namesToDelete, name)
			}
		}
	}

	//delete merged values
	for _, name := range namesToDelete {
		delete(result, name)
	}

	//sort and set the merged names with single list of values
	for nameWithoutNr, nrsWithValues := range merged {
		if values, ok := result[nameWithoutNr]; ok {
			nrsWithValues = append(nrsWithValues, nrWithValues{nr: 0, values: values}) //if also defined without _#
		}
		sort.Slice(nrsWithValues, func(i, j int) bool { return nrsWithValues[i].nr < nrsWithValues[j].nr })

		list := []string{}
		for _, nrWithValues := range nrsWithValues {
			list = append(list, nrWithValues.values...)
		}
		result[nameWithoutNr] = list
	}
	return result
}

type nrWithValues struct {
	nr     int64
	values []string
}

//converts query string params to named values that can be parsed into a struct
//it support both single/multi-value params, depending how you get them from your HTTP library
//    (e.g. AWS API Gateway Context returns both but default golang net/http returns only params)
func NamedValuesFromURL(params map[string]string, multiValueParams map[string][]string) map[string][]string {
	result := map[string][]string{}
	for n, v := range params {
		result[n] = []string{v}
	}
	for n, mv := range multiValueParams {
		if list, ok := result[n]; !ok {
			result[n] = mv
		} else {
			//do not add duplicates - seems like AWS put same value in both single and multivalue params
			for _, v := range mv {
				found := false
				for _, existingValue := range list {
					if v == existingValue {
						found = true
					}
				}
				if !found {
					list = append(list, v)
				}
			}
			result[n] = list
		}
	}

	for name, values := range result {
		splitValues := []string{}
		for _, value := range values {
			//split only if valid CSV between [...]
			if value[0] == '[' && value[len(value)-1] == ']' {
				csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
				csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
				if csvErr == nil {
					splitValues = append(splitValues, csvValues...)
				} else {
					splitValues = append(splitValues, value) //cannot split this "[...]" value
				}
			} else {
				splitValues = append(splitValues, value) //not a "[...]" value
			}
		}
		result[name] = splitValues
	}
	return result
}

// Purpose:
// 	UnmarshalNamedValues() parses a set of named values into a struct using json tag matching
//  Unlike json.Unmarshal(), it takes care of converting quoted "true" -> true, "1" -> int(1) etc...
//
//	Typically used to parse environment or URL params into a struct
//	because normal json.Unmarshal() will fail to parse "2" into an integer etc
//
//	By convention, the names should be lowercase to match json tag with "_" delimeters
//	And also use "_" for nested sub-struct names
//	  named value "a_b_c_d":5 would be stored in
//    field with json tag "a_b_c_d" or
//	  field with json tag "a_b"        which is a struct with a json tagged field "c_d" etc...
//
// Parameters:
// 	namedValues is name-value pairs, typical from URL params or OS environment
//		see construction functions for this:
//			NamedValuesFromEnv()
//			NamedValuesFromURL()
//
// 	structPtr must be ptr to a struct variable
//		undefined values will not be changed, so you can call this multiple times on the
//		same struct to amend a few values, leaving the rest and default values unchanged
//
// Return:
//	unused values
// 	nil or error if some values could not be used
//
//	If all values must be used, check len(unusedValues) when err==nil
//
func UnmarshalNamedValues(namedValues map[string][]string, structPtr interface{}) (unusedValues map[string][]string, err error) {
	if structPtr == nil {
		return nil, errors.Errorf("cannot unmarshal into nil")
	}
	structPtrType := reflect.TypeOf(structPtr)
	if structPtrType.Kind() != reflect.Ptr || structPtrType.Elem().Kind() != reflect.Struct {
		return nil, errors.Errorf("%T is not &struct", structPtr)
	}
	structType := structPtrType.Elem()
	structPtrValue := reflect.ValueOf(structPtr)

	if usedNameList, err := unmarshalNamedValuesIntoStructPtr("", namedValues, structType, structPtrValue); err != nil {
		return namedValues, err
	} else {
		for _, usedName := range usedNameList {
			delete(namedValues, usedName)
		}
	}
	return namedValues, nil
}

func unmarshalNamedValuesIntoStructPtr(prefix string, namedValues map[string][]string, structType reflect.Type, structPtrValue reflect.Value) (usedNameList []string, err error) {
	usedNameList = []string{}
	for i := 0; i < structType.NumField(); i++ {
		structTypeField := structType.Field(i)
		fieldName := (strings.SplitN(structTypeField.Tag.Get("json"), ",", 2))[0]
		if fieldName == "-" {
			continue //skip fields excluded from JSON
		}
		if prefix != "" {
			if fieldName != "" {
				fieldName = prefix + "_" + fieldName
			} else {
				fieldName = prefix
			}
		}

		//recurse into anonymous sub-structs
		if structTypeField.Type.Kind() == reflect.Struct {
			if nameList, err := unmarshalNamedValuesIntoStructPtr(fieldName, namedValues, structTypeField.Type, structPtrValue.Elem().Field(i).Addr()); err != nil {
				return nil, errors.Wrapf(err, "failed on %s.%s", structType.Name(), fieldName)
			} else {
				usedNameList = append(usedNameList, nameList...)
			}
			continue
		}

		fieldValues, ok := namedValues[fieldName]
		if !ok {
			continue //skip undefined fields
		}
		usedNameList = append(usedNameList, fieldName)
		delete(namedValues, fieldName)
		if len(fieldValues) == 0 {
			continue //field has no value specified in URL, do not remove values not defined (cannot clear defined struct fields with named values)
		}
		structPtrFieldValue := structPtrValue.Elem().Field(i)
		if structPtrFieldValue.Kind() == reflect.Ptr {
			//this is a ptr, allocate a new value and set it
			//then we can dereference to set it below
			structPtrFieldValue.Set(reflect.New(structPtrFieldValue.Type().Elem()))
			structPtrFieldValue = structPtrFieldValue.Elem()
		}

		//param is defined >=1 times in URL
		if structTypeField.Type.Kind() == reflect.Slice {
			//this param struct field is a slice, iterate over all specified values
			for i, fieldValue := range fieldValues {
				parsedValue, parseErr := unmarshalValue(fieldValue, structTypeField.Type.Elem())
				if parseErr != nil {
					err = errors.Wrapf(parseErr, "invalid %s[%d]", fieldName, i)
					return
				}
				structPtrFieldValue.Set(reflect.Append(structPtrFieldValue, parsedValue))

				//todo: sorting of list using value names as applicable
			}
		} else {
			//field is not a slice, expecting only a single value
			if len(fieldValues) > 1 {
				err = errors.Errorf("%s cannot store multiple value (%d found: %+v)", fieldName, len(fieldValues), fieldValues)
				return
			}
			parsedValue, parseErr := unmarshalValue(fieldValues[0], structPtrFieldValue.Type())
			if parseErr != nil {
				err = errors.Wrapf(parseErr, "invalid %s", fieldName)
				return
			}
			structPtrFieldValue.Set(parsedValue)
		}
	} //for each param struct field
	return usedNameList, nil
}

func unmarshalValue(v interface{}, t reflect.Type) (reflect.Value, error) {
	newValuePtr := reflect.New(t)
	if reflect.ValueOf(v).Type().AssignableTo(t) {
		newValuePtr.Elem().Set(reflect.ValueOf(v)) //can assign as is
	} else {
		//needs conversion
		s, ok := v.(string)
		if !ok {
			jsonValue, _ := json.Marshal(v)
			s = string(jsonValue)
		}
		//is string value, unmarshal as quoted or unquoted JSON value
		if err := json.Unmarshal([]byte("\""+s+"\""), newValuePtr.Interface()); err != nil {
			if err := json.Unmarshal([]byte(s), newValuePtr.Interface()); err != nil {
				return newValuePtr.Elem(), errors.Wrapf(err, "invalid \"%s\"", s)
			}
		}
	}
	return newValuePtr.Elem(), nil
}