Skip to content
Snippets Groups Projects
Select Git revision
  • 00b0546be819518c1d1582ba7f791c310b826b76
  • main default protected
  • v1.302.0
  • v1.301.0
  • v1.300.0
  • v1.299.0
  • v1.298.0
  • v1.297.0
  • v1.296.0
  • v1.295.0
  • v1.294.0
  • v1.293.0
  • v1.292.0
  • v1.291.0
  • v1.290.0
  • v1.289.0
  • v1.288.0
  • v1.287.0
  • v1.286.0
  • v1.285.0
  • v1.284.0
  • v1.283.0
22 results

named_values_to_struct.go

Blame
  • named_values_to_struct.go 11.01 KiB
    package struct_utils
    
    import (
    	"database/sql"
    	"encoding/csv"
    	"encoding/json"
    	"reflect"
    	"sort"
    	"strconv"
    	"strings"
    
    	"gitlab.com/uafrica/go-utils/errors"
    	"gitlab.com/uafrica/go-utils/logger"
    	"gitlab.com/uafrica/go-utils/string_utils"
    )
    
    //Purpose:
    //	Make a list of named values from the env for parsing into a struct
    //
    //Parameters:
    //	prefix should be uppercase (by convention) env prefix like "MY_LIB_CONFIG", without trailing "_"
    //
    //Result:
    //	named values that can be passed into UnmarshalNamedValues()
    //
    //All env starting with "<prefix>_" will be copied without "<prefix>_"
    //Examples with prefix="MY_LIB_CONFIG":
    // MY_LIB_CONFIG_MAX_SIZE="6"        -> {"MAX_SIZE":["6"]}			one value of "6"
    // MY_LIB_CONFIG_NAMES   ="A,B,C"    -> {"NAMES":["A,B,C"]}         one value of "A,B,C"
    // MY_LIB_CONFIG_NRS     ="1,2,3"    -> {"NRS":["1,2,3"]}           one value of "1,2,3" (all env values are string, later parsed into int based on struct field type)
    // MY_LIB_CONFIG_CODES   ="[1,2,3]"" -> {"CODES":["1","2","3"]}     3 values of "1", "2" and "3" because of outer [...], env values are string
    //
    // MY_LIB_CONFIG_CODES_1=5
    // MY_LIB_CONFIG_CODES_5=7
    // MY_LIB_CONFIG_CODES_2=10          -> {"CODES":["5","10","7"]}	3 values ordered on suffixes "_1", "_5", "_2" moving 10 before 7
    //
    // MY_LIB_CONFIG_ADDRS=["55 Crescent, Town", "12 Big Street, City"] -> 2 values including commas because of quoted CSV
    func NamedValuesFromEnv(prefix string) map[string][]string {
    	return NamedValuesFromReader(prefix, string_utils.EnvironmentKeyReader())
    }
    
    func NamedValuesFromReader(prefix string, reader string_utils.KeyReader) map[string][]string {
    	if reader == nil {
    		return nil
    	}
    	result := map[string][]string{}
    	prefix += "_"
    	for _, key := range reader.Keys(prefix) {
    		value, ok := reader.GetString(key)
    		key = key[len(prefix):]
    		if !ok {
    			logger.Debugf("Key(%s) undefined", key)
    			continue
    		}
    		logger.Debugf("key(%s)=\"%s\"", key, value)
    		result[strings.ToLower(key)] = []string{value}
    
    		//split only if valid CSV between [...]
    		if value[0] == '[' && value[len(value)-1] == ']' {
    			csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
    			csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
    			if csvErr == nil {
    				result[strings.ToLower(key)] = csvValues
    			}
    		}
    	}
    
    	//merge multiple <name>_#=<value> into single lists called <name>
    	namesToDelete := []string{}
    	merged := map[string][]nrWithValues{}
    	for name, values := range result {
    		delimIndex := strings.LastIndex(name, "_")
    		if delimIndex > 0 {
    			nr := name[delimIndex+1:]
    			if i64, err := strconv.ParseInt(nr, 10, 64); err == nil {
    				nameWithoutNr := name[:delimIndex]
    				if _, ok := merged[nameWithoutNr]; !ok {
    					merged[nameWithoutNr] = []nrWithValues{}
    				}
    				merged[nameWithoutNr] = append(merged[nameWithoutNr], nrWithValues{nr: i64, values: values})
    				namesToDelete = append(namesToDelete, name)
    			}
    		}
    	}
    
    	//delete merged values
    	for _, name := range namesToDelete {
    		delete(result, name)
    	}
    
    	//sort and set the merged names with single list of values
    	for nameWithoutNr, nrsWithValues := range merged {
    		if values, ok := result[nameWithoutNr]; ok {
    			nrsWithValues = append(nrsWithValues, nrWithValues{nr: 0, values: values}) //if also defined without _#
    		}
    		sort.Slice(nrsWithValues, func(i, j int) bool { return nrsWithValues[i].nr < nrsWithValues[j].nr })
    
    		list := []string{}
    		for _, nrWithValues := range nrsWithValues {
    			list = append(list, nrWithValues.values...)
    		}
    		result[nameWithoutNr] = list
    	}
    	return result
    }
    
    type nrWithValues struct {
    	nr     int64
    	values []string
    }
    
    //converts query string params to named values that can be parsed into a struct
    //it support both single/multi-value params, depending how you get them from your HTTP library
    //    (e.g. AWS API Gateway Context returns both but default golang net/http returns only params)
    func NamedValuesFromURL(params map[string]string, multiValueParams map[string][]string) map[string][]string {
    	result := map[string][]string{}
    	for n, v := range params {
    		result[n] = []string{v}
    	}
    	for n, mv := range multiValueParams {
    		if list, ok := result[n]; !ok {
    			result[n] = mv
    		} else {
    			//do not add duplicates - seems like AWS put same value in both single and multivalue params
    			for _, v := range mv {
    				found := false
    				for _, existingValue := range list {
    					if v == existingValue {
    						found = true
    					}
    				}
    				if !found {
    					list = append(list, v)
    				}
    			}
    			result[n] = list
    		}
    	}
    
    	for name, values := range result {
    		splitValues := []string{}
    		for _, value := range values {
    			//split only if valid CSV between [...]
    			if value[0] == '[' && value[len(value)-1] == ']' {
    				csvReader := csv.NewReader(strings.NewReader(value[1 : len(value)-1]))
    				csvValues, csvErr := csvReader.Read() //this automatically removes quotes around some/all CSV inside the [...]
    				if csvErr == nil {
    					splitValues = append(splitValues, csvValues...)
    				} else {
    					splitValues = append(splitValues, value) //cannot split this "[...]" value
    				}
    			} else {
    				splitValues = append(splitValues, value) //not a "[...]" value
    			}
    		}
    		result[name] = splitValues
    	}
    	return result
    }
    
    // Purpose:
    // 	UnmarshalNamedValues() parses a set of named values into a struct using json tag matching
    //  Unlike json.Unmarshal(), it takes care of converting quoted "true" -> true, "1" -> int(1) etc...
    //
    //	Typically used to parse environment or URL params into a struct
    //	because normal json.Unmarshal() will fail to parse "2" into an integer etc
    //
    //	By convention, the names should be lowercase to match json tag with "_" delimeters
    //	And also use "_" for nested sub-struct names
    //	  named value "a_b_c_d":5 would be stored in
    //    field with json tag "a_b_c_d" or
    //	  field with json tag "a_b"        which is a struct with a json tagged field "c_d" etc...
    //
    // Parameters:
    // 	namedValues is name-value pairs, typical from URL params or OS environment
    //		see construction functions for this:
    //			NamedValuesFromEnv()
    //			NamedValuesFromURL()
    //
    // 	structPtr must be ptr to a struct variable
    //		undefined values will not be changed, so you can call this multiple times on the
    //		same struct to amend a few values, leaving the rest and default values unchanged
    //
    // Return:
    //	unused values
    // 	nil or error if some values could not be used
    //
    //	If all values must be used, check len(unusedValues) when err==nil
    //
    func UnmarshalNamedValues(namedValues map[string][]string, structPtr interface{}) (unusedValues map[string][]string, err error) {
    	if structPtr == nil {
    		return nil, errors.Errorf("cannot unmarshal into nil")
    	}
    	structPtrType := reflect.TypeOf(structPtr)
    	if structPtrType.Kind() != reflect.Ptr || structPtrType.Elem().Kind() != reflect.Struct {
    		return nil, errors.Errorf("%T is not &struct", structPtr)
    	}
    	structType := structPtrType.Elem()
    	structPtrValue := reflect.ValueOf(structPtr)
    
    	if usedNameList, err := unmarshalNamedValuesIntoStructPtr("", namedValues, structType, structPtrValue); err != nil {
    		return namedValues, err
    	} else {
    		for _, usedName := range usedNameList {
    			delete(namedValues, usedName)
    		}
    	}
    	return namedValues, nil
    }
    
    func unmarshalNamedValuesIntoStructPtr(prefix string, namedValues map[string][]string, structType reflect.Type, structPtrValue reflect.Value) (usedNameList []string, err error) {
    	usedNameList = []string{}
    	for i := 0; i < structType.NumField(); i++ {
    		structTypeField := structType.Field(i)
    		fieldName := (strings.SplitN(structTypeField.Tag.Get("json"), ",", 2))[0]
    		if fieldName == "-" {
    			continue //skip fields excluded from JSON
    		}
    		if prefix != "" {
    			if fieldName != "" {
    				fieldName = prefix + "_" + fieldName
    			} else {
    				fieldName = prefix
    			}
    		}
    
    		//recurse into anonymous sub-structs
    		if structTypeField.Type.Kind() == reflect.Struct {
    			if nameList, err := unmarshalNamedValuesIntoStructPtr(fieldName, namedValues, structTypeField.Type, structPtrValue.Elem().Field(i).Addr()); err != nil {
    				return nil, errors.Wrapf(err, "failed on %s.%s", structType.Name(), fieldName)
    			} else {
    				usedNameList = append(usedNameList, nameList...)
    			}
    			continue
    		}
    
    		fieldValues, ok := namedValues[fieldName]
    		if !ok {
    			continue //skip undefined fields
    		}
    		usedNameList = append(usedNameList, fieldName)
    		delete(namedValues, fieldName)
    		if len(fieldValues) == 0 {
    			continue //field has no value specified in URL, do not remove values not defined (cannot clear defined struct fields with named values)
    		}
    		structPtrFieldValue := structPtrValue.Elem().Field(i)
    		if structPtrFieldValue.Kind() == reflect.Ptr {
    			//this is a ptr, allocate a new value and set it
    			//then we can dereference to set it below
    			structPtrFieldValue.Set(reflect.New(structPtrFieldValue.Type().Elem()))
    			structPtrFieldValue = structPtrFieldValue.Elem()
    		}
    
    		//param is defined >=1 times in URL
    		if structTypeField.Type.Kind() == reflect.Slice {
    			//this param struct field is a slice, iterate over all specified values
    			for i, fieldValue := range fieldValues {
    				parsedValue, parseErr := unmarshalValue(fieldValue, structTypeField.Type.Elem())
    				if parseErr != nil {
    					err = errors.Wrapf(parseErr, "invalid %s[%d]", fieldName, i)
    					return
    				}
    				structPtrFieldValue.Set(reflect.Append(structPtrFieldValue, parsedValue))
    
    				//todo: sorting of list using value names as applicable
    			}
    		} else {
    			//field is not a slice, expecting only a single value
    			if len(fieldValues) > 1 {
    				err = errors.Errorf("%s cannot store multiple value (%d found: %+v)", fieldName, len(fieldValues), fieldValues)
    				return
    			}
    			parsedValue, parseErr := unmarshalValue(fieldValues[0], structPtrFieldValue.Type())
    			if parseErr != nil {
    				err = errors.Wrapf(parseErr, "invalid %s", fieldName)
    				return
    			}
    			structPtrFieldValue.Set(parsedValue)
    		}
    	} //for each param struct field
    	return usedNameList, nil
    }
    
    func unmarshalValue(v interface{}, t reflect.Type) (reflect.Value, error) {
    	newValuePtr := reflect.New(t)
    	if reflect.ValueOf(v).Type().AssignableTo(t) {
    		newValuePtr.Elem().Set(reflect.ValueOf(v)) //can assign as is
    		return newValuePtr.Elem(), nil
    	}
    
    	//needs conversion
    	s, ok := v.(string)
    	if !ok {
    		jsonValue, _ := json.Marshal(v)
    		s = string(jsonValue)
    	}
    
    	//now we have string value
    	if valueScanner, ok := newValuePtr.Interface().(sql.Scanner); ok {
    		//if has scanner - prefer that over json unmarshal
    		//because we do not know if json expects quoted/unquoted for this type
    		//and if we try quoted, it fail, then try unquoted, we have two different
    		//errors one one of them will be kind of meaning nothing, but which?
    		//scanner should always take the value as typed, so let's try that first
    		if err := valueScanner.Scan(s); err == nil {
    			return newValuePtr.Elem(), nil
    		}
    	}
    
    	//try JSON unmarshal as is else with quotes
    	if err := json.Unmarshal([]byte(s), newValuePtr.Interface()); err == nil {
    		return newValuePtr.Elem(), nil
    	}
    	if err := json.Unmarshal([]byte("\""+s+"\""), newValuePtr.Interface()); err != nil {
    		return newValuePtr.Elem(), errors.Wrapf(err, "invalid \"%s\"", s)
    	}
    	return newValuePtr.Elem(), nil
    }