diff --git a/.gitignore b/.gitignore
index 474488e4d645d28352bce17353bff11cce21e57f..fb6d4c2be0a3fb37b59a6e1c196deb5e65fa330d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,6 @@
# Dependency directories (remove the comment below to include it)
# vendor/
.idea
+examples/core/api/api
+examples/core/cron/cron
+examples/core/sqs/sqs
diff --git a/api/README.md b/api/README.md
index 010c529cf90d9feaa49d2e37b4b2c878c83fdcc3..07d1cb94ef67f150d4d34462e3128f7226c15e83 100644
--- a/api/README.md
+++ b/api/README.md
@@ -1,25 +1,23 @@
# TO TEST
- mage run and local run
-- claims
-- ctx values, read from handler and set in handler then retrieve later in handler
+- claims *& impersonate
# TODO
-- sqs
-- cron
-- combined local service with api,sqs and cron
+- crash dump recover does not log the call stack - impossible to debug
+- sqs local & cloud
+- cron local & cloud
- use in v3 and shiplogic
-- config for local running
-- db connection
-
-
-- when handler returns an error to indicate HTTP code, then it is wrapped and user gets:
-
-```{"message":"error from handler, because user not found HTTP(404:Not Found), because user does not exist"}``` instead of just:
-```{"message":"user does not exist"}```
-
+- config for local running - from cdk stuff...
+- db connection in app + claims in app
- api-docs not yet working here - and need to provide HTML option or per-endpoint options at least
+- log as JSON when not running local
+- remove log clutter from API but allow switch on/off or dump on error using log sync...
+- log as JSON - see if crash dump etc are clearly logged as fields, not as part of a string
+# Later
- add path parameters, e.g. /user/{user_id} - make sure it works the same way as lambda does - but we're not using it at present, I assume because the old simple map[][] router did not support it.
-- load config for local running
- document with examples and templates
-- scheduled tasks from the db or some other scheduler from AWS?
\ No newline at end of file
+- scheduled tasks from the db or some other scheduler from AWS?
+- clone db locally then run with local db not to mess up dev for others
+
+- API test sequences configure & part of docs
diff --git a/api/api.go b/api/api.go
index 5fcf8ca2a6d8c390533fe8dcd143726efa6d9c30..63d3dbe9671802dc5d7599d881dfd2132d75f198 100644
--- a/api/api.go
+++ b/api/api.go
@@ -3,19 +3,18 @@ package api
import (
"fmt"
"net/http"
- "os"
- "regexp"
+ "runtime/debug"
"sync"
- "time"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
+ "gitlab.com/uafrica/go-utils/audit"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
queues_mem "gitlab.com/uafrica/go-utils/queues/mem"
queues_sqs "gitlab.com/uafrica/go-utils/queues/sqs"
"gitlab.com/uafrica/go-utils/service"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
//LEGACY: global variable is set only for backward compatibility
@@ -26,10 +25,6 @@ var CurrentRequestID *string
//value could be any of the handler function signatures supported by the api.Router
//requestIDHeaderKey is defined in the response header to match the requestID from the request
func New(requestIDHeaderKey string, routes map[string]map[string]interface{}) Api {
- env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
- if env == "" {
- env = "dev"
- }
if requestIDHeaderKey == "" {
requestIDHeaderKey = "request-id"
}
@@ -40,87 +35,85 @@ func New(requestIDHeaderKey string, routes map[string]map[string]interface{}) Ap
}
return Api{
- ILogger: logger.New().WithFields(map[string]interface{}{"env": env}),
- env: env,
- router: router,
- requestIDHeaderKey: requestIDHeaderKey,
- checks: []check{},
- crashReporter: defaultCrashReporter{},
- auditor: noAudit{},
+ Service: service.New(),
+ router: router,
+ requestIDHeaderKey: requestIDHeaderKey,
+ checks: map[string]ICheck{},
+ crashReporter: defaultCrashReporter{},
+ cors: nil,
+ localPort: 0,
+ localQueueEventHandlers: nil,
}
}
type Api struct {
- logger.ILogger //for logging outside of context
- env string
- router Router
- requestIDHeaderKey string
-
- //options:
- localPort int //==0 for lambda, >0 for http.ListenAndServe
- eventHandlers map[string]interface{}
- crashReporter ICrashReporter
- cors ICORS
- producer queues.IProducer
- dbConn service.IDatabaseConnector
- checks []check
- auditor IAuditor
+ service.Service
+ router Router
+ requestIDHeaderKey string
+ checks map[string]ICheck
+ crashReporter ICrashReporter
+ cors ICORS
+ localPort int //==0 for default lambda, >0 for http.ListenAndServe to run locally
+ localQueueEventHandlers map[string]interface{} //only applies when running locally for local in-memory queues
}
-//checks are executed at the start of each request
-//to stop processing, return an error, preferably with an HTTP status code (see errors.HTTP(...))
-//all values returned will be logged and added to the context for retrieval by any handler
-//the values are added with the check's name as prefix: "<check.name>_<value.name>"
-//and cannot be changed afterwards
-type IChecker interface {
- Check(req events.APIGatewayProxyRequest) (values map[string]interface{}, err error)
+//wrap Service.WithStarter to return api, else cannot be chained
+func (api Api) WithStarter(name string, starter service.IStarter) Api {
+ api.Service = api.Service.WithStarter(name, starter)
+ return api
}
-type check struct {
- name string
- checker IChecker
+//wrap Service.WithErrorReporter to return api, else cannot be chained
+func (api Api) WithErrorReporter(reporter service.IErrorReporter) Api {
+ api.Service = api.Service.WithErrorReporter(reporter)
+ return api
}
-const namePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+//wrap else cannot be chained
+func (api Api) WithAuditor(auditor audit.Auditor) Api {
+ api.Service = api.Service.WithAuditor(auditor)
+ return api
+}
-var nameRegex = regexp.MustCompile("^" + namePattern + "$")
+//wrap else cannot be chained
+func (api Api) WithProducer(producer service.Producer) Api {
+ api.Service = api.Service.WithProducer(producer)
+ return api
+}
-//check name must be lowercase with optional underscores, e.g. "rate_limiter" or "claims"
-func (app Api) WithCheck(name string, checker IChecker) Api {
- if !nameRegex.MatchString(name) {
- panic(errors.Errorf("invalid name for check(%s)", name))
+//add a check to startup of each context
+//they will be called in the sequence they were added
+//if check return error, processing stops and err is returned
+//if check succeed, and return !=nil data, it is stored against the name
+// so your handler can retieve it with:
+// checkData := ctx.Value(name).(expectedType)
+// or
+// checkData,ok := ctx.Value(name).(expectedType)
+// if !ok { ... }
+//you can implement one check that does everything and return a struct or
+//implement one for your db, one for rate limit, one for auth, one for ...
+//the name must be snake-case, e.g. "this_is_my_check_data_name"
+func (api Api) WithCheck(name string, check ICheck) Api {
+ if !string_utils.IsSnakeCase(name) {
+ panic(errors.Errorf("invalid check name=\"%s\", expecting snake_case names only", name))
}
- if checker == nil {
- panic(errors.Errorf("check(%s) is nil", name))
+ if check == nil {
+ panic(errors.Errorf("check(%s) func==nil", name))
}
- for _, check := range app.checks {
- if check.name == name {
- panic(errors.Errorf("check(%s) already registered", name))
- }
+ if _, ok := api.checks[name]; ok {
+ panic(errors.Errorf("check(%s) already defined", name))
}
- app.checks = append(app.checks, check{name: name, checker: checker})
- return app
-}
-
-func (api Api) WithDb(dbConn service.IDatabaseConnector) Api {
- api.dbConn = dbConn
+ api.checks[name] = check
return api
}
func (api Api) WithCORS(cors ICORS) Api {
- api.cors = cors
- return api
-}
-
-func (api Api) WithProducer(producer queues.IProducer) Api {
- api.producer = producer
+ if cors != nil {
+ api.cors = cors
+ }
return api
}
-type ICORS interface {
- CORS() map[string]string //return CORS headers
-}
-
func (api Api) WithCrashReported(crashReporter ICrashReporter) Api {
if crashReporter != nil {
api.crashReporter = crashReporter
@@ -128,15 +121,14 @@ func (api Api) WithCrashReported(crashReporter ICrashReporter) Api {
return api
}
-type ICrashReporter interface {
- Catch(ctx Context) //Report(method string, path string, crash interface{})
-}
-
func (api Api) WithLocalPort(localPortPtr *int, eventHandlers map[string]interface{}) Api {
+ if api.localPort != 0 {
+ panic("local port already defined")
+ }
if localPortPtr != nil && *localPortPtr > 0 {
api.localPort = *localPortPtr
+ api.localQueueEventHandlers = eventHandlers
}
- api.eventHandlers = eventHandlers
return api
}
@@ -146,20 +138,20 @@ func (api Api) Run() {
if api.localPort > 0 {
//running locally with standard HTTP server
- if api.eventHandlers != nil {
+ if api.localQueueEventHandlers != nil {
//when running locally - we want to send and process SQS events locally using channels
//here we create a SQS chan and start listening to it
//again: this is quick hack... will make this part of framework once it works well
api.Debugf("Creating local queue consumer/producer...")
- memConsumer := queues_mem.NewConsumer(api.eventHandlers)
- api.producer = queues_mem.NewProducer(memConsumer)
+ memConsumer := queues_mem.NewConsumer(api.localQueueEventHandlers)
+ api = api.WithProducer(queues_mem.NewProducer(memConsumer))
sqsEventChan := make(chan events.SQSEvent)
sqsWaitGroup := sync.WaitGroup{}
sqsWaitGroup.Add(1)
go func() {
for event := range sqsEventChan {
- logger.Debug("NOT YET PROCESSING SQS Event: %+v", event)
+ logger.Debugf("NOT YET PROCESSING SQS Event: %+v", event)
}
sqsWaitGroup.Done()
}()
@@ -171,32 +163,24 @@ func (api Api) Run() {
}()
} else {
//use SQS for events
- api.producer = queues_sqs.NewProducer(api.requestIDHeaderKey)
+ api = api.WithProducer(queues_sqs.NewProducer(api.requestIDHeaderKey))
}
- err := http.ListenAndServe(fmt.Sprintf(":%d", api.localPort), api) //calls app.ServeHTTP() which calls app.Handler()
+ err := http.ListenAndServe(fmt.Sprintf(":%d", api.localPort), api) //calls api.ServeHTTP() which calls api.Handler()
if err != nil {
panic(err)
}
} else {
- api.producer = queues_sqs.NewProducer(api.requestIDHeaderKey)
- lambda.Start(api.Handler) //calls app.Handler directly
+ api = api.WithProducer(queues_sqs.NewProducer(api.requestIDHeaderKey))
+ lambda.Start(api.Handler) //calls api.Handler directly
}
}
type defaultCrashReporter struct{}
func (defaultCrashReporter) Catch(ctx Context) {
- // crash := recover()
- // if crash != nil {
- // ctx.Errorf("CRASH: (%T) %+v\n", crash, crash)
- // }
-}
-
-type IAuditor interface {
- Audit(startTime, endTime time.Time, values map[string]interface{})
+ crashErr := recover()
+ if crashErr != nil {
+ ctx.Errorf("crashed: %v, with stack: %s", crashErr, string(debug.Stack()))
+ }
}
-
-type noAudit struct{}
-
-func (noAudit) Audit(startTime, endTime time.Time, values map[string]interface{}) {} //do nothing
diff --git a/api/check.go b/api/check.go
new file mode 100644
index 0000000000000000000000000000000000000000..b7be7f618670557360ab03ef3dd1cb685e61abad
--- /dev/null
+++ b/api/check.go
@@ -0,0 +1,5 @@
+package api
+
+type ICheck interface {
+ Check(Context) (interface{}, error)
+}
diff --git a/api/context.go b/api/context.go
index ec961f0e43ded2b0629d785900922002048f4370..f031af756b5adc7f8c1c68570ef72eacab4b165f 100644
--- a/api/context.go
+++ b/api/context.go
@@ -1,96 +1,61 @@
package api
import (
- "context"
"encoding/json"
- "fmt"
"reflect"
- "strconv"
"strings"
- "time"
"github.com/aws/aws-lambda-go/events"
- "github.com/uptrace/bun"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
+ "gitlab.com/uafrica/go-utils/reflection"
"gitlab.com/uafrica/go-utils/service"
)
-type IContext interface {
- context.Context
- logger.ILogger
- queues.IProducer
- StartTime() time.Time
- MillisecondsSinceStart() int64
- CheckValues(checkName string) interface{}
- CheckValue(checkName, valueName string) interface{}
-}
-
-type Context struct {
+type Context interface {
service.Context
- queues.IProducer
- Request events.APIGatewayProxyRequest
- RequestID string
- ValuesFromChecks map[string]map[string]interface{} //also in context.Value(), but cannot retrieve iteratively from there for logging...
- DB *bun.DB
+ Request() events.APIGatewayProxyRequest
}
-func (ctx Context) CheckValues(checkName string) interface{} {
- if cv, ok := ctx.ValuesFromChecks[checkName]; ok {
- return cv
- }
- return nil
-}
+var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
-func (ctx Context) CheckValue(checkName, valueName string) interface{} {
- if cv, ok := ctx.ValuesFromChecks[checkName]; ok {
- if v, ok := cv[valueName]; ok {
- return v
- }
- }
- return nil
+type apiContext struct {
+ service.Context
+ request events.APIGatewayProxyRequest
}
-// func (ctx Context) Audit(org, new interface{}, eventType types.AuditEventType) {
-// //call old function for now - should become part of context ONLY
-// audit.SaveAuditEvent(org, new, ctx.Claims, eventType, &ctx.RequestID)
-// }
+func (ctx apiContext) Request() events.APIGatewayProxyRequest {
+ return ctx.request
+}
//todo: change to be a ctx method that defer to log so it does not have to be called explicitly
//it should also capture metrics for the handler and automaticlaly write the audit record,
//(but still allow for audit to be suppressed may be in some cases)
-func (ctx Context) LogAPIRequestAndResponse(res events.APIGatewayProxyResponse, err error) {
+func (ctx *apiContext) LogAPIRequestAndResponse(res events.APIGatewayProxyResponse, err error) {
fields := map[string]interface{}{
- "path": ctx.Request.Path,
- "method": ctx.Request.HTTPMethod,
+ "path": ctx.request.Path,
+ "method": ctx.request.HTTPMethod,
"status_code": res.StatusCode,
- "api_gateway_request_id": ctx.RequestID,
+ "api_gateway_request_id": ctx.RequestID(),
}
- if ctx.Request.HTTPMethod == "GET" {
- fields["req-query"] = ctx.Request.QueryStringParameters
+ if ctx.request.HTTPMethod == "GET" {
+ fields["req-query"] = ctx.request.QueryStringParameters
}
statusOK := res.StatusCode >= 200 && res.StatusCode <= 299
if err != nil || !statusOK {
fields["error"] = err
- fields["req-body"] = ctx.Request.Body
- fields["req-query"] = ctx.Request.QueryStringParameters
+ fields["req-body"] = ctx.request.Body
+ fields["req-query"] = ctx.request.QueryStringParameters
fields["res-body"] = res.Body
- for checkName, checkValues := range ctx.ValuesFromChecks {
- for name, value := range checkValues {
- fields[checkName+"_"+name] = value
- }
- }
}
- ctx.WithFields(fields).Infof("Request & Response: err=%+v", err)
+ ctx.Context.WithFields(fields).Infof("Request & Response: err=%+v", err)
}
//allocate struct for params, populate it from the URL parameters then validate and return the struct
-func (ctx Context) GetRequestParams(paramsStructType reflect.Type) (interface{}, error) {
+func (ctx apiContext) GetRequestParams(paramsStructType reflect.Type) (interface{}, error) {
paramValues := map[string]interface{}{}
- for n, v := range ctx.Request.QueryStringParameters {
+ for n, v := range ctx.request.QueryStringParameters {
paramValues[n] = v
}
paramsStructValuePtr := reflect.New(paramsStructType)
@@ -106,10 +71,10 @@ func (ctx Context) GetRequestParams(paramsStructType reflect.Type) (interface{},
//get value(s) from query string
var paramStrValues []string
- if paramStrValue, isDefined := ctx.Request.QueryStringParameters[n]; isDefined {
+ if paramStrValue, isDefined := ctx.request.QueryStringParameters[n]; isDefined {
paramStrValues = []string{paramStrValue} //single value
} else {
- paramStrValues = ctx.Request.MultiValueQueryStringParameters[n]
+ paramStrValues = ctx.request.MultiValueQueryStringParameters[n]
}
if len(paramStrValues) == 0 {
continue //param has no value specified in URL
@@ -120,11 +85,8 @@ func (ctx Context) GetRequestParams(paramsStructType reflect.Type) (interface{},
//iterate over all specified values
for index, paramStrValue := range paramStrValues {
newValuePtr := reflect.New(f.Type.Elem())
- if err := setParamFromStr(fmt.Sprintf("%s[%d]", n, index),
- //paramsStructValuePtr.Elem().Field(i).Index(index),
- newValuePtr.Elem(), //.Elem() to dereference
- paramStrValue); err != nil {
- return nil, errors.Wrapf(err, "failed to set %s[%d]=%s", n, i, paramStrValues[0])
+ if err := reflection.SetValue(newValuePtr.Elem(), paramStrValue); err != nil {
+ return nil, errors.Wrapf(err, "failed to set %s[%d]=%s", n, index, paramStrValues[0])
}
paramsStructValuePtr.Elem().Field(i).Set(reflect.Append(paramsStructValuePtr.Elem().Field(i), newValuePtr.Elem()))
}
@@ -133,12 +95,16 @@ func (ctx Context) GetRequestParams(paramsStructType reflect.Type) (interface{},
return nil, errors.Errorf("%s does not support >1 values(%v)", n, strings.Join(paramStrValues, ","))
}
//single value specified
- if err := setParamFromStr(n, paramsStructValuePtr.Elem().Field(i), paramStrValues[0]); err != nil {
+ if err := reflection.SetValue(paramsStructValuePtr.Elem().Field(i), paramStrValues[0]); err != nil {
return nil, errors.Wrapf(err, "failed to set %s=%s", n, paramStrValues[0])
}
}
} //for each param struct field
+ if err := ctx.applyClaim("params", paramsStructValuePtr.Interface()); err != nil {
+ return nil, errors.Wrapf(err, "failed to fill claims on params")
+ }
+
if validator, ok := paramsStructValuePtr.Interface().(IValidator); ok {
if err := validator.Validate(); err != nil {
return nil, errors.Wrapf(err, "invalid params")
@@ -148,83 +114,16 @@ func (ctx Context) GetRequestParams(paramsStructType reflect.Type) (interface{},
return paramsStructValuePtr.Elem().Interface(), nil
}
-func setParamFromStr(fieldName string, fieldValue reflect.Value, paramStrValue string) error {
- logger.Debugf("Set(%s,%v,%v,%v)", fieldName, fieldValue.Type(), fieldValue.Kind(), paramStrValue)
- switch fieldValue.Type().Kind() {
- case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
- //parse to int for this struct field
- if i64, err := strconv.ParseInt(paramStrValue, 10, 64); err != nil {
- return errors.Errorf("%s is not a number", paramStrValue)
- } else {
- switch fieldValue.Type().Kind() {
- case reflect.Int:
- fieldValue.Set(reflect.ValueOf(int(i64)))
- case reflect.Int8:
- fieldValue.Set(reflect.ValueOf(int8(i64)))
- case reflect.Int16:
- fieldValue.Set(reflect.ValueOf(int16(i64)))
- case reflect.Int32:
- fieldValue.Set(reflect.ValueOf(int32(i64)))
- case reflect.Int64:
- fieldValue.Set(reflect.ValueOf(i64))
- }
- }
-
- case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
- //parse to int for this struct field
- if u64, err := strconv.ParseUint(paramStrValue, 10, 64); err != nil {
- return errors.Errorf("%s is not a number", paramStrValue)
- } else {
- switch fieldValue.Type().Kind() {
- case reflect.Uint:
- fieldValue.Set(reflect.ValueOf(uint(u64)))
- case reflect.Uint8:
- fieldValue.Set(reflect.ValueOf(uint8(u64)))
- case reflect.Uint16:
- fieldValue.Set(reflect.ValueOf(uint16(u64)))
- case reflect.Uint32:
- fieldValue.Set(reflect.ValueOf(uint32(u64)))
- case reflect.Uint64:
- fieldValue.Set(reflect.ValueOf(u64))
- }
- }
-
- case reflect.Bool:
- bs := strings.ToLower(paramStrValue)
- if bs == "true" || bs == "yes" || bs == "1" {
- fieldValue.Set(reflect.ValueOf(true))
- }
-
- case reflect.String:
- fieldValue.Set(reflect.ValueOf(paramStrValue))
-
- case reflect.Float32:
- if f32, err := strconv.ParseFloat(paramStrValue, 32); err != nil {
- return errors.Wrapf(err, "invalid float")
- } else {
- fieldValue.Set(reflect.ValueOf(float32(f32)))
- }
-
- case reflect.Float64:
- if f64, err := strconv.ParseFloat(paramStrValue, 64); err != nil {
- return errors.Wrapf(err, "invalid float")
- } else {
- fieldValue.Set(reflect.ValueOf(f64))
- }
-
- default:
- return errors.Errorf("unsupported type %v", fieldValue.Type().Kind())
- } //switch param struct field
- return nil
-}
-
-func (ctx Context) GetRequestBody(requestStructType reflect.Type) (interface{}, error) {
+func (ctx apiContext) GetRequestBody(requestStructType reflect.Type) (interface{}, error) {
requestStructValuePtr := reflect.New(requestStructType)
- err := json.Unmarshal([]byte(ctx.Request.Body), requestStructValuePtr.Interface())
+ err := json.Unmarshal([]byte(ctx.request.Body), requestStructValuePtr.Interface())
if err != nil {
return nil, errors.Wrapf(err, "failed to JSON request body")
}
+ if err := ctx.applyClaim("body", requestStructValuePtr.Interface()); err != nil {
+ return nil, errors.Wrapf(err, "failed to fill claims on body")
+ }
if validator, ok := requestStructValuePtr.Interface().(IValidator); ok {
if err := validator.Validate(); err != nil {
return nil, errors.Wrapf(err, "invalid request body")
@@ -237,3 +136,52 @@ func (ctx Context) GetRequestBody(requestStructType reflect.Type) (interface{},
type IValidator interface {
Validate() error
}
+
+func (ctx *apiContext) applyClaim(name string, valuePtr interface{}) error {
+ t := reflect.TypeOf(valuePtr)
+ if t.Kind() != reflect.Ptr {
+ return errors.Errorf("%T is not a pointer", valuePtr) //programming error... it must be a pointer to be able to change it
+ }
+ t = t.Elem()
+ if t.Kind() != reflect.Struct {
+ //ctx.Debugf("Not setting claims on %T", valuePtr)
+ return nil //not a struct - nothing to do - is allowed e.g. for posting a string.
+ }
+ if err := ctx.setClaim(name, t, reflect.ValueOf(valuePtr).Elem()); err != nil {
+ return errors.Wrapf(err, "failed to set claim on %T", valuePtr)
+ }
+ return nil
+}
+
+func (ctx *apiContext) setClaim(name string, structType reflect.Type, structValue reflect.Value) error {
+ for fieldName, claimValue := range ctx.Claim() {
+ if field := structValue.FieldByName(fieldName); field.IsValid() {
+ if err := reflection.SetValue(field, claimValue); err != nil {
+ return errors.Errorf("failed to set %s.%s=(%T)%v", structType.Name(), fieldName, claimValue, claimValue)
+ }
+ ctx.Debugf("defined claim %s.%s=(%T)%v ...", name, fieldName, claimValue, claimValue)
+ } /* else {
+ ctx.Debugf("claim(%s) does not apply to %s", fieldName, structType.Name())
+ }*/
+ }
+
+ //recurse into sub-structs and sub struct ptrs (not yet slices)
+ for i := 0; i < structType.NumField(); i++ {
+ f := structType.Field(i)
+ if len(f.Name) > 0 && f.Name[0] >= 'a' && f.Name[0] <= 'z' {
+ //private field - do not enter
+ continue
+ }
+ if f.Type.Kind() == reflect.Struct {
+ if err := ctx.setClaim(name+"."+structType.Field(i).Name, f.Type, structValue.Field(i)); err != nil {
+ return errors.Wrapf(err, "failed to set claim on sub struct %s.%s", structType.Name(), f.Name)
+ }
+ }
+ if f.Type.Kind() == reflect.Ptr && f.Type.Elem().Kind() == reflect.Struct && !structValue.Field(i).IsNil() {
+ if err := ctx.setClaim(name+"."+structType.Field(i).Name, f.Type.Elem(), structValue.Field(i).Elem()); err != nil {
+ return errors.Wrapf(err, "failed to set claim on sub &struct %s.%s", structType.Name(), f.Name)
+ }
+ }
+ }
+ return nil
+}
diff --git a/api/cors.go b/api/cors.go
new file mode 100644
index 0000000000000000000000000000000000000000..4d36770c56c41ccb1f309904ca2803c338ad9065
--- /dev/null
+++ b/api/cors.go
@@ -0,0 +1,5 @@
+package api
+
+type ICORS interface {
+ CORS() map[string]string //return CORS headers
+}
diff --git a/api/crash.go b/api/crash.go
new file mode 100644
index 0000000000000000000000000000000000000000..bf686eac384417f9bf0eae14a7d6e805abb81125
--- /dev/null
+++ b/api/crash.go
@@ -0,0 +1,5 @@
+package api
+
+type ICrashReporter interface {
+ Catch(ctx Context) //Report(method string, path string, crash interface{})
+}
diff --git a/api/handler.go b/api/handler.go
index 8e4e785684117135fe33f1a6b3c980d4b21f7365..3aa1cebe7317eba0b6775a4e3e536499bfd62236 100644
--- a/api/handler.go
+++ b/api/handler.go
@@ -24,9 +24,12 @@ func NewHandler(fnc interface{}) (handler, error) {
return h, errors.Errorf("returns %d results instead of ([Response,] error)", fncType.NumOut())
}
- //arg[0] must implement interface lambda_helpers.Context
- if _, ok := reflect.New(fncType.In(0)).Interface().(IContext); !ok {
- return h, errors.Errorf("first arg %v does not implement api.IContext", fncType.In(0))
+ //arg[0] must implement interface api.Context
+ //if _, ok := reflect.New(fncType.In(0)).Interface().(Context); !ok {
+
+ if fncType.In(0) != contextInterfaceType &&
+ !fncType.In(0).Implements(contextInterfaceType) {
+ return h, errors.Errorf("first arg %v does not implement %v", fncType.In(0), contextInterfaceType)
}
//arg[1] must be a struct for params. It may be an empty struct, but
diff --git a/api/lambda.go b/api/lambda.go
index 5f4836ebf9e6f1376a2a52256cdfdf11fda8fed9..f4fe02762a5585b87441dcddd8aa0be9f9a2cb56 100644
--- a/api/lambda.go
+++ b/api/lambda.go
@@ -2,6 +2,7 @@ package api
import (
"context"
+ "database/sql"
"encoding/json"
"fmt"
"math/rand"
@@ -12,7 +13,7 @@ import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambdacontext"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/service"
+ "gitlab.com/uafrica/go-utils/logger"
)
//this is native handler for lambda passed into lambda.Start()
@@ -30,24 +31,30 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
apiGatewayProxyReq.Resource = apiGatewayProxyReq.Path
}
- //setup context
- requestID := apiGatewayProxyReq.RequestContext.RequestID
- if lambdaContext, ok := lambdacontext.FromContext(baseCtx); ok && lambdaContext != nil {
- requestID = lambdaContext.AwsRequestID
+ //get request-id from HTTP headers (used when making internal service calls)
+ //if not defined in header, get the AWS request id from the AWS context
+ requestID, ok := apiGatewayProxyReq.Headers[api.requestIDHeaderKey]
+ if !ok || requestID == "" {
+ if lambdaContext, ok := lambdacontext.FromContext(baseCtx); ok && lambdaContext != nil {
+ requestID = lambdaContext.AwsRequestID
+ }
}
- ctx := Context{
- Context: service.NewContext(baseCtx, map[string]interface{}{
- "env": api.env,
- "request_id": requestID,
- }),
- IProducer: api.producer,
- Request: apiGatewayProxyReq,
- RequestID: requestID,
- ValuesFromChecks: map[string]map[string]interface{}{},
+
+ //service context invoke the starters and could fail, e.g. if cannot connect to db
+ serviceContext, err := api.Service.NewContext(baseCtx, requestID, nil)
+ if err != nil {
+ return res, err
+ }
+
+ ctx := &apiContext{
+ Context: serviceContext,
+ request: apiGatewayProxyReq,
}
//report handler crashes
- defer api.crashReporter.Catch(ctx)
+ if api.crashReporter != nil {
+ defer api.crashReporter.Catch(ctx)
+ }
defer func() {
//set CORS headers on every response
@@ -61,48 +68,38 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
defer func() {
ctx.LogAPIRequestAndResponse(res, err)
if err != nil {
+ ctx.Errorf("failed: %+v", err)
+
+ //try to retrieve HTTP code from error
if withCause, ok := err.(errors.ErrorWithCause); ok && withCause.Code() != 0 {
res.StatusCode = withCause.Code()
+ err = withCause.Cause() //drop the http layers + up
+ } else {
+ //no HTTP code indicate in err,
+ //see if there are SQL errors that could indicate code
+ if code, ok := StatusCodeFromSQLError(err); ok {
+ res.StatusCode = code
+ }
}
+
errorMessage := fmt.Sprintf("%c", err)
jsonError, _ := json.Marshal(map[string]interface{}{"message": errorMessage})
+ res.Headers["Content-Type"] = "application/json"
res.Body = string(jsonError)
+ api.Service.ReportError(ctx.Data(), err)
err = nil //never pass error back to lambda or http server
}
if api.requestIDHeaderKey != "" {
- res.Headers[api.requestIDHeaderKey] = ctx.RequestID
+ res.Headers[api.requestIDHeaderKey] = ctx.RequestID()
}
- api.auditor.Audit(ctx.StartTime(), time.Now(), map[string]interface{}{
- "request_id": ctx.RequestID,
- "request": ctx.Request,
+ if err := api.Service.WriteValues(ctx.StartTime(), time.Now(), ctx.RequestID(), map[string]interface{}{
+ "request_id": ctx.RequestID(),
+ "request": ctx.request,
"response": res},
- )
- }()
-
- if api.dbConn != nil {
- ctx.DB, err = api.dbConn.Connect()
- if err != nil {
- err = errors.Wrapf(err, "failed to connect to db")
- return
- }
- }
-
- //do checks before proceed (may use db connection)
- //(typical maintenance mode, rate limits, authentication/claims, ...)
- for _, check := range api.checks {
- var checkValues map[string]interface{}
- checkValues, err = check.checker.Check(apiGatewayProxyReq)
- //for n, v := range checkValues {
- //ctx.Context = ctx.Context.WithValue(check.name+"_"+n, v)
- //ctx.Debugf("Defined value[%s]=(%T)%v", check.name+"_"+n, v, v)
- //}
- ctx.ValuesFromChecks[check.name] = checkValues
- ctx.Debugf("Defined ValuesFromChecks[%s]=(%T)%v", check.name, checkValues, checkValues)
- if err != nil {
- err = errors.Wrapf(err, "check(%s) failed", check.name)
- return
+ ); err != nil {
+ ctx.Errorf("failed to audit: %+v", err)
}
- }
+ }()
//Early return OPTIONS call
if apiGatewayProxyReq.HTTPMethod == "OPTIONS" {
@@ -113,17 +110,30 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
rand.Seed(time.Now().Unix())
+ for checkName, check := range api.checks {
+ var checkData interface{}
+ checkData, err = check.Check(ctx)
+ if err != nil {
+ err = errors.Wrapf(err, "%s", checkName)
+ return
+ }
+ if err = ctx.Set(checkName, checkData); err != nil {
+ err = errors.Wrapf(err, "failed to set check(%s) data=(%T)%+v", checkName, checkData, checkData)
+ return
+ }
+ }
+
//LEGACY: delete this as soon as all handlers accepts context
//this does not support concurrent execution!
- CurrentRequestID = &ctx.Request.RequestContext.RequestID
+ CurrentRequestID = &ctx.request.RequestContext.RequestID
ctx.Debugf("HTTP %s %s ...\n", apiGatewayProxyReq.HTTPMethod, apiGatewayProxyReq.Resource)
ctx.WithFields(map[string]interface{}{
- "http_method": ctx.Request.HTTPMethod,
- "path": ctx.Request.Path,
- "api_gateway_request_id": ctx.Request.RequestContext.RequestID,
- "user_cognito_auth_provider": ctx.Request.RequestContext.Identity.CognitoAuthenticationProvider,
- "user_arn": ctx.Request.RequestContext.Identity.UserArn,
+ "http_method": ctx.request.HTTPMethod,
+ "path": ctx.request.Path,
+ "api_gateway_request_id": ctx.request.RequestContext.RequestID,
+ "user_cognito_auth_provider": ctx.request.RequestContext.Identity.CognitoAuthenticationProvider,
+ "user_arn": ctx.request.RequestContext.Identity.UserArn,
}).Infof("Start API Handler")
//TODO:
@@ -181,7 +191,7 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
return
}
//apply claims to params struct - TODO: Removed - see if cannot force to get claims from context always
- // if err = ctx.Claims.FillOnObject(ctx.Request, ¶msStruct); err != nil {
+ // if err = ctx.Claims.FillOnObject(ctx.request, ¶msStruct); err != nil {
// err = errors.HTTP(http.StatusInternalServerError, err, "claims failed on parameters")
// return
// }
@@ -202,7 +212,7 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
}
//apply claims to request struct - TODO: Removed - see if cannot force to get claims from context always
- // if err = ctx.Claims.FillOnObject(ctx.Request, &bodyStruct); err != nil {
+ // if err = ctx.Claims.FillOnObject(ctx.request, &bodyStruct); err != nil {
// err = errors.HTTP(http.StatusInternalServerError, err, "claims failed on body")
// return
// }
@@ -212,12 +222,13 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
}
//call handler in a func with defer to catch potential crash
- ctx.Debugf("calling handler")
+ ctx.Infof("Calling handle %s %s ...", apiGatewayProxyReq.HTTPMethod, apiGatewayProxyReq.Resource)
var results []reflect.Value
results, err = func() (results []reflect.Value, err error) {
defer func() {
- if crashDump := recover(); crashDump != nil {
- err = errors.Errorf("handler function crashed: %+v", crashDump)
+ if crashErr := recover(); crashErr != nil {
+ stack := logger.CallStack()
+ err = errors.Errorf("handler function crashed: %v, with stack: %+v", crashErr, stack)
return
}
}()
@@ -229,14 +240,12 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
return
}
- ctx.Debugf("handler -> results: %v", results)
-
+ //ctx.Debugf("handler -> results: %v", results)
//see if handler failed using last result of type error
lastResultValue := results[len(results)-1]
if !lastResultValue.IsNil() {
err = lastResultValue.Interface().(error)
if err != nil {
- err = errors.Wrapf(err, "error from handler")
return
}
}
@@ -255,7 +264,22 @@ func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGat
err = errors.Wrapf(err, "failed to encode response content")
return
}
+ res.Headers["Content-Type"] = "application/json"
res.Body = string(bodyBytes)
}
return
}
+
+//look for SQL errors in the error stack
+func StatusCodeFromSQLError(err error) (int, bool) {
+ if err == sql.ErrNoRows {
+ return http.StatusNotFound, true
+ }
+
+ if errWithCause, ok := err.(errors.ErrorWithCause); ok {
+ return StatusCodeFromSQLError(errWithCause.Cause())
+ }
+
+ //could not determine known SQL error
+ return 0, false
+}
diff --git a/api/local.go b/api/local.go
index fb4b14293d35cdf90bf868b288c74b2a0da1373c..0aa903a5b342883e7fe57fcf2ccfd2c22370df13 100644
--- a/api/local.go
+++ b/api/local.go
@@ -13,6 +13,7 @@ import (
//use this in http.ListenAndServe() to test locally
func (api Api) ServeHTTP(httpRes http.ResponseWriter, httpReq *http.Request) {
+ api.Debugf("HTTP %s %s", httpReq.Method, httpReq.URL.Path)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
@@ -88,12 +89,14 @@ func (api Api) ServeHTTP(httpRes http.ResponseWriter, httpReq *http.Request) {
httpRes.Header().Set(n, v)
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
- http.Error(httpRes, res.Body, res.StatusCode)
+ httpRes.Header().Set("X-Content-Type-Options", "nosniff")
+ httpRes.Header().Set("Content-Type", "application/json")
+ httpRes.WriteHeader(res.StatusCode)
+ httpRes.Write([]byte(res.Body)) //assuming this is a JSON error message
return
}
if res.Body != "" {
- httpRes.Header().Set("Content-Type", "application/json")
httpRes.Write([]byte(res.Body))
}
}
diff --git a/audit/audit.go b/audit/audit.go
new file mode 100644
index 0000000000000000000000000000000000000000..1a7f3710bd5f0a7039931947ef83060f1805bbca
--- /dev/null
+++ b/audit/audit.go
@@ -0,0 +1,146 @@
+package audit
+
+import "time"
+
+type Auditor interface {
+ WriteValues(startTime, endTime time.Time, requestID string, values map[string]interface{}) error
+ WriteEvent(requestID string, event Event) error
+}
+
+// func getAuthUsername(identity events.APIGatewayRequestIdentity) string {
+// if identity.CognitoAuthenticationProvider != "" {
+// split := strings.Split(identity.CognitoAuthenticationProvider, ":")
+// return split[len(split)-1]
+// }
+
+// // IAM
+// split := strings.Split(identity.UserArn, ":user/")
+// return split[len(split)-1]
+// }
+
+// func SaveAPIRequest(req events.APIGatewayProxyRequest, response events.APIGatewayProxyResponse, startTime time.Time, currentRequestID *string) {
+// claim, err := claims.RetrieveClaims(&req)
+// authType := determineAuthType(req.RequestContext.Identity)
+// authUsername := getAuthUsername(req.RequestContext.Identity)
+
+// responseBody := response.Body
+// responseSize := int64(len(responseBody))
+// // SQS has a 256KB limit, so let's limit the response to 96KB
+// if len(responseBody) > 12000 {
+// responseBody = responseBody[:12000] + "..."
+// }
+
+// var relevantID *string
+// mappedResponse := map[string]interface{}{}
+// err = json.Unmarshal([]byte(req.Body), &mappedResponse)
+
+// if err == nil {
+// val, present := mappedResponse["id"]
+// if present {
+// valString := utils.Int64ToString(int64(val.(float64)))
+// relevantID = &valString
+// }
+// }
+
+// accountID := fmt.Sprintf("%d", *claim.AccountID)
+// apiLog := types.ApiLog{
+// AccountID: &accountID,
+// UserID: claim.UserID,
+// Path: req.Path,
+// HTTPMethod: req.HTTPMethod,
+// Timestamp: time.Now(),
+// Request: map[string]interface{}{
+// "Body": req.Body,
+// "QueryStringParameters": req.QueryStringParameters,
+// },
+// Response: map[string]interface{}{
+// "Body": responseBody,
+// },
+// ResponseCode: response.StatusCode,
+// ExecutionTime: time.Now().Sub(startTime).Milliseconds(),
+// RequestID: currentRequestID,
+// RelevantID: relevantID,
+// InitialAuthUsername: &authUsername,
+// InitialAuthType: authType,
+// IP: &req.RequestContext.Identity.SourceIP,
+// UserAgent: &req.RequestContext.Identity.UserAgent,
+// ResponseSize: responseSize,
+// }
+
+// data, err := json.Marshal(apiLog)
+// if err != nil {
+// logs.LogErrorMessage("Failed to encode audit event", err)
+// return
+// }
+
+// _ = sqs.ApiLogEvent(
+// map[string]string{},
+// string(data),
+// currentRequestID,
+// )
+// }
+
+// func SaveThirdPartyAPIRequest(url string, method string, requestBody string, responseBody string, responseStatus int, startTime time.Time, currentRequestID *string) {
+
+// responseSize := int64(len(responseBody))
+// // SQS has a 256KB limit, so let's limit the response to 96KB
+// if len(responseBody) > 12000 {
+// responseBody = responseBody[:12000] + "..."
+// }
+
+// var relevantID *string
+// mappedResponse := map[string]interface{}{}
+// err := json.Unmarshal([]byte(requestBody), &mappedResponse)
+
+// if err == nil {
+// val, present := mappedResponse["id"]
+// if present {
+// valString := utils.Int64ToString(int64(val.(float64)))
+// relevantID = &valString
+// }
+// }
+
+// apiLog := types.ApiLog{
+// AccountID: nil,
+// UserID: nil,
+// Path: url,
+// HTTPMethod: method,
+// Timestamp: time.Now(),
+// Request: map[string]interface{}{
+// "Body": requestBody,
+// },
+// Response: map[string]interface{}{
+// "Body": responseBody,
+// },
+// ResponseCode: responseStatus,
+// ExecutionTime: time.Now().Sub(startTime).Milliseconds(),
+// RequestID: currentRequestID,
+// RelevantID: relevantID,
+// InitialAuthUsername: nil,
+// InitialAuthType: nil,
+// IP: nil,
+// UserAgent: nil,
+// ResponseSize: responseSize,
+// }
+
+// data, err := json.Marshal(apiLog)
+// if err != nil {
+// logs.LogErrorMessage("Failed to encode audit event", err)
+// return
+// }
+
+// _ = sqs.ApiLogEvent(
+// map[string]string{},
+// string(data),
+// currentRequestID,
+// )
+// }
+
+// func determineAuthType(identity events.APIGatewayRequestIdentity) *string {
+// result := "cognito"
+// if identity.CognitoAuthenticationType == "" {
+// result = "iam"
+// }
+
+// return &result
+// }
diff --git a/audit/event.go b/audit/event.go
new file mode 100644
index 0000000000000000000000000000000000000000..dc61743472d209b2f5d99e39027e908a7198d0e7
--- /dev/null
+++ b/audit/event.go
@@ -0,0 +1,177 @@
+package audit
+
+import (
+ "fmt"
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/r3labs/diff/v2"
+ "gitlab.com/uafrica/go-utils/reflection"
+)
+
+type Event struct {
+ ID int64 `json:"id"`
+ ObjectID string `json:"object_id"`
+ Type string `json:"type"`
+ Source string `json:"source"`
+ Timestamp time.Time `json:"timestamp"`
+ Change map[string]interface{} `json:"change"`
+}
+
+//parameters:
+// source could be "" then defaults to "SYSTEM" or specify the user name that made the change
+// orgValue and newValue could be nil
+// they are compared and changes are logged
+func NewEvent(source string, eventType string, orgValue, newValue interface{}) (Event, error) {
+ changelog, err := diff.Diff(orgValue, newValue)
+ if err != nil {
+ return Event{}, err
+ }
+
+ changes := map[string]interface{}{}
+ for _, change := range changelog {
+ if len(change.Path) == 1 {
+ // Root object change
+ field := ToSnakeCase(change.Path[0])
+ changes[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ } else if len(change.Path) == 2 {
+ // Child object changed
+ // ["Account", "ID"]
+ // 0 = Object
+ // 1 = field
+
+ objectKey := ToSnakeCase(change.Path[0])
+ field := ToSnakeCase(change.Path[1])
+
+ existingObject, present := changes[objectKey]
+ if present {
+ if object, ok := existingObject.(map[string]interface{}); ok {
+ object[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ changes[objectKey] = object
+ }
+ } else {
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = fieldChange
+ }
+
+ } else if len(change.Path) == 3 {
+ objectKey := ToSnakeCase(change.Path[0])
+ indexString := change.Path[1]
+ index, _ := strconv.ParseInt(indexString, 10, 64)
+ field := ToSnakeCase(change.Path[2])
+ arrayObject, present := changes[objectKey]
+ if present {
+ if arrayOfObjects, ok := arrayObject.([]map[string]interface{}); ok {
+ if len(arrayOfObjects) > int(index) {
+ // Add field to existing object in array
+ object := arrayOfObjects[index]
+ object[field] = FieldChange{
+ From: change.From,
+ To: change.To,
+ }
+ } else {
+ // new object, append to existing array
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = append(arrayOfObjects, fieldChange)
+ }
+
+ }
+ } else {
+ // Create array of objects
+ fieldChange := map[string]interface{}{
+ field: FieldChange{
+ From: change.From,
+ To: change.To,
+ },
+ }
+ changes[objectKey] = []map[string]interface{}{
+ fieldChange,
+ }
+ }
+ }
+ }
+
+ objectID := getIntValue(orgValue, "ID")
+ if objectID == 0 {
+ objectID = getIntValue(newValue, "ID")
+ }
+
+ objectIDString := fmt.Sprintf("%v", objectID)
+ if objectIDString == "0" {
+ objectIDString = getStringValue(orgValue, "Username")
+ }
+ if objectIDString == "" {
+ objectIDString = getStringValue(newValue, "Username")
+ }
+ if objectIDString == "" {
+ objectIDString = getStringValue(orgValue, "Key")
+ }
+ if objectIDString == "" {
+ objectIDString = getStringValue(newValue, "Key")
+ }
+
+ event := Event{
+ ObjectID: objectIDString,
+ Source: source,
+ Type: eventType,
+ Timestamp: time.Now(),
+ Change: changes,
+ }
+ if event.Source == "" {
+ event.Source = "SYSTEM"
+ }
+ return event, nil
+}
+
+type FieldChange struct {
+ From interface{} `json:"change_from"`
+ To interface{} `json:"change_to"`
+}
+
+var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
+var matchAllCap = regexp.MustCompile("([a-z0-9])([A-Z])")
+
+func ToSnakeCase(str string) string {
+ snake := matchFirstCap.ReplaceAllString(str, "${1}_${2}")
+ snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
+ return strings.ToLower(snake)
+}
+
+func getIntValue(object interface{}, key string) int64 {
+ structValue := reflect.ValueOf(object)
+ if structValue.Kind() == reflect.Struct {
+ field := structValue.FieldByName(key)
+ id := reflection.GetInt64Value(field)
+ return id
+ }
+ return 0
+}
+
+func getStringValue(object interface{}, key string) string {
+ structValue := reflect.ValueOf(object)
+ if structValue.Kind() == reflect.Struct {
+ field := structValue.FieldByName(key)
+ id := reflection.GetStringValue(field)
+ return id
+ }
+ return ""
+}
diff --git a/audit/file-audit.go b/audit/file-audit.go
new file mode 100644
index 0000000000000000000000000000000000000000..176851dbc2a1864e6fba8e6416f50c2d3a38b8f9
--- /dev/null
+++ b/audit/file-audit.go
@@ -0,0 +1,59 @@
+package audit
+
+import (
+ "encoding/json"
+ "os"
+ "time"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+//creates auditor that writes to file, which could be os.Stderr or os.Stdout for debugging
+func File(f *os.File) Auditor {
+ if f == nil {
+ panic(errors.Errorf("cannot create file auditor with f=nil"))
+ }
+ return fileAudit{
+ f: f,
+ }
+}
+
+type fileAudit struct {
+ f *os.File
+}
+
+func (fa fileAudit) WriteValues(startTime, endTime time.Time, requestID string, values map[string]interface{}) error {
+ obj := map[string]interface{}{
+ "start_time": startTime,
+ "end_time": endTime,
+ "duration": endTime.Sub(startTime),
+ "request_id": requestID,
+ "values": values,
+ }
+ jsonObj, err := json.Marshal(obj)
+ if err != nil {
+ return errors.Wrapf(err, "failed to JSON encode audit values")
+ }
+ if _, err := fa.f.Write(jsonObj); err != nil {
+ return errors.Wrapf(err, "failed to write audit values to file")
+ }
+ return nil
+}
+
+func (fa fileAudit) WriteEvent(requestID string, event Event) error {
+ obj := map[string]interface{}{
+ "start_time": event.Timestamp,
+ "end_time": event.Timestamp,
+ "duration": 0,
+ "request_id": requestID,
+ "values": event,
+ }
+ jsonObj, err := json.Marshal(obj)
+ if err != nil {
+ return errors.Wrapf(err, "failed to JSON encode audit event")
+ }
+ if _, err := fa.f.Write(jsonObj); err != nil {
+ return errors.Wrapf(err, "failed to write audit event to file")
+ }
+ return nil
+}
diff --git a/audit/no-audit.go b/audit/no-audit.go
new file mode 100644
index 0000000000000000000000000000000000000000..930f4134f7c4f2a820bea8ca64e6982bd90d4656
--- /dev/null
+++ b/audit/no-audit.go
@@ -0,0 +1,20 @@
+package audit
+
+import (
+ "time"
+)
+
+//creates auditor that writes nothiong
+func None() Auditor {
+ return noAudit{}
+}
+
+type noAudit struct{}
+
+func (noAudit) WriteValues(startTime, endTime time.Time, requestID string, values map[string]interface{}) error {
+ return nil
+}
+
+func (noAudit) WriteEvent(requestID string, event Event) error {
+ return nil
+}
diff --git a/cron/check.go b/cron/check.go
new file mode 100644
index 0000000000000000000000000000000000000000..71a3e5317bc451b47c889a26da23c79fb7c51705
--- /dev/null
+++ b/cron/check.go
@@ -0,0 +1,5 @@
+package cron
+
+type ICheck interface {
+ Check(Context) (interface{}, error)
+}
diff --git a/cron/context.go b/cron/context.go
index 734dbcd220e9593e53c9ff63c100146a1bf360d3..7903e70e8c29690d5d2a3f97a94f852f806af6c4 100644
--- a/cron/context.go
+++ b/cron/context.go
@@ -2,22 +2,34 @@ package cron
import (
"context"
- "time"
+ "reflect"
- "github.com/uptrace/bun"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/service"
)
-type IContext interface {
- context.Context
- logger.ILogger
- StartTime() time.Time
- MillisecondsSinceStart() int64
+type Context interface {
+ service.Context
}
-type Context struct {
+var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
+
+type cronContext struct {
service.Context
- Name string //cron function name
- DB *bun.DB
+ name string //cron function name
+}
+
+func (cron Cron) NewContext(baseCtx context.Context, requestID string, cronName string) (Context, error) {
+ serviceContext, err := cron.Service.NewContext(baseCtx, requestID, map[string]interface{}{
+ "cron": cronName,
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create service context")
+ }
+
+ ctx := cronContext{
+ Context: serviceContext,
+ name: cronName,
+ }
+ return ctx, nil
}
diff --git a/cron/cron.go b/cron/cron.go
index 0683c14646278dd8aaafad68bc2a693363fb65c5..abcfe25e33f2177c91a4be8d116f5d8e520277d3 100644
--- a/cron/cron.go
+++ b/cron/cron.go
@@ -8,9 +8,11 @@ import (
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-lambda-go/lambdacontext"
"github.com/google/uuid"
+ "gitlab.com/uafrica/go-utils/audit"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
"gitlab.com/uafrica/go-utils/service"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
func New(functions map[string]func(Context) error) Cron {
@@ -25,22 +27,69 @@ func New(functions map[string]func(Context) error) Cron {
}
return Cron{
- ILogger: logger.New().WithFields(map[string]interface{}{"env": env}),
+ Service: service.New(),
+ Logger: logger.New().WithFields(map[string]interface{}{"env": env}),
env: env,
router: router,
+ checks: map[string]ICheck{},
crashReporter: defaultCrashReporter{},
}
}
type Cron struct {
- logger.ILogger
+ service.Service
+ logger.Logger
env string
router Router
- dbConn service.IDatabaseConnector
+ checks map[string]ICheck
crashReporter ICrashReporter
}
-func (cron Cron) WithCrashReported(crashReporter ICrashReporter) Cron {
+//wrap Service.WithStarter to return cron, else cannot be chained
+func (cron Cron) WithStarter(name string, starter service.IStarter) Cron {
+ cron.Service = cron.Service.WithStarter(name, starter)
+ return cron
+}
+
+//wrap Service.WithErrorReporter to return api, else cannot be chained
+func (cron Cron) WithErrorReporter(reporter service.IErrorReporter) Cron {
+ cron.Service = cron.Service.WithErrorReporter(reporter)
+ return cron
+}
+
+//wrap else cannot be chained
+func (cron Cron) WithAuditor(auditor audit.Auditor) Cron {
+ cron.Service = cron.Service.WithAuditor(auditor)
+ return cron
+}
+
+//add a check to startup of each context
+//they will be called in the sequence they were added
+//if check return error, processing stops and err is returned
+//if check succeed, and return !=nil data, it is stored against the name
+// so your handler can retieve it with:
+// checkData := ctx.Value(name).(expectedType)
+// or
+// checkData,ok := ctx.Value(name).(expectedType)
+// if !ok { ... }
+//you can implement one check that does everything and return a struct or
+//implement one for your db, one for rate limit, one for auth, one for ...
+//the name must be snake-case, e.g. "this_is_my_check_data_name"
+func (cron Cron) WithCheck(name string, check ICheck) Cron {
+ if !string_utils.IsSnakeCase(name) {
+ panic(errors.Errorf("invalid check name=\"%s\", expecting snake_case names only", name))
+ }
+ if check == nil {
+ panic(errors.Errorf("check(%s) func==nil", name))
+ }
+ if _, ok := cron.checks[name]; ok {
+ panic(errors.Errorf("check(%s) already defined", name))
+ }
+ cron.checks[name] = check
+ return cron
+}
+
+func (cron Cron) WithCrashReporter(crashReporter ICrashReporter) Cron {
if crashReporter != nil {
cron.crashReporter = crashReporter
}
@@ -60,11 +109,6 @@ func (defaultCrashReporter) Catch(ctx Context) {
// }
}
-func (cron Cron) WithDb(dbConn service.IDatabaseConnector) Cron {
- cron.dbConn = dbConn
- return cron
-}
-
func (cron Cron) Run(invokeArn *string) {
if invokeArn != nil && *invokeArn != "" {
//just run this handler and terminate - for testing on a terminal
@@ -81,9 +125,10 @@ func (cron Cron) Run(invokeArn *string) {
)
err := cron.Handler(lambdaContext)
if err != nil {
- panic(errors.Errorf("cron failed: %+v", err))
+ logger.Errorf("local cron handler failed: %+v", err)
+ //cron.Service.ReportError(nil, err)
} else {
- logger.Debugf("cron success")
+ logger.Debugf("local cron success")
}
return
}
diff --git a/cron/handler.go b/cron/handler.go
index dd3772bbfc261435d79ca0d97437b2d7f47d3939..956e14cb3d20992cfcacac4b34dc1f58a1e7b108 100644
--- a/cron/handler.go
+++ b/cron/handler.go
@@ -22,9 +22,10 @@ func NewHandler(fnc interface{}) (Handler, error) {
return h, errors.Errorf("returns %d results instead of (error)", fncType.NumOut())
}
- //arg[0] must implement interface sqs.IContext
- if _, ok := reflect.New(fncType.In(0)).Interface().(IContext); !ok {
- return h, errors.Errorf("first arg %v does not implement sqs.IContext", fncType.In(0))
+ //arg[0] must implement interface queues.Context
+ if fncType.In(0) != contextInterfaceType &&
+ !fncType.In(0).Implements(contextInterfaceType) {
+ return h, errors.Errorf("first arg %v does not implement %v", fncType.In(0), contextInterfaceType)
}
//arg[1] must be a struct for the message record body. It may be an empty struct, but
diff --git a/cron/lambda.go b/cron/lambda.go
index ea2fb6fd3ca09ffa74bc989b6f7b2d37e4b136a8..3835f08975f4a565c116942b80ef179897ab828f 100644
--- a/cron/lambda.go
+++ b/cron/lambda.go
@@ -7,7 +7,6 @@ import (
"github.com/aws/aws-lambda-go/lambdacontext"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/service"
)
type LambdaCronHandler func(lambdaCtx context.Context) error
@@ -24,13 +23,10 @@ func (cron Cron) Handler(lambdaCtx context.Context) (err error) {
//got a handler, prepare to run:
rand.Seed(time.Now().Unix())
- ctx := Context{
- Context: service.NewContext(lambdaCtx, map[string]interface{}{
- "env": cron.env,
- "request-id": requestID,
- "cron": cronName,
- }),
- Name: cronName,
+ //service context invoke the starters and could fail, e.g. if cannot connect to db
+ ctx, err := cron.NewContext(lambdaCtx, requestID, cronName)
+ if err != nil {
+ return err
}
defer func() {
@@ -39,21 +35,29 @@ func (cron Cron) Handler(lambdaCtx context.Context) (err error) {
}
}()
- if cron.dbConn != nil {
- ctx.DB, err = cron.dbConn.Connect()
+ //report handler crashes
+ if cron.crashReporter != nil {
+ defer cron.crashReporter.Catch(ctx)
+ }
+
+ for checkName, check := range cron.checks {
+ var checkData interface{}
+ checkData, err = check.Check(ctx)
if err != nil {
- err = errors.Wrapf(err, "failed to connect to db")
+ err = errors.Wrapf(err, "%s", checkName)
+ return
+ }
+ if err = ctx.Set(checkName, checkData); err != nil {
+ err = errors.Wrapf(err, "failed to set check(%s) data=(%T)%+v", checkName, checkData, checkData)
return
}
}
- //report handler crashes
- defer cron.crashReporter.Catch(ctx)
-
//todo: set log level, trigger log on conditions, sync at end of transaction - after log level was determined
ctx.Infof("Start CRON Handler")
if err := cronFunc(ctx); err != nil {
+ cron.Service.ReportError(ctx.Data(), err)
return errors.Wrapf(err, "Cron(%s) failed", cronName)
}
return nil
diff --git a/errors/caller.go b/errors/caller.go
index 3e29bf3d9d2ed1d3e7a8d25d91f42aee9b18d8ed..f3ef9cac480b9f232770e3c50bdba3b4a69204f8 100644
--- a/errors/caller.go
+++ b/errors/caller.go
@@ -23,6 +23,14 @@ type Caller struct {
pkgDotFunc string
}
+func NewCaller(pkgDotFunc string, file string, line int) Caller {
+ return Caller{
+ pkgDotFunc: pkgDotFunc,
+ file: file,
+ line: line,
+ }
+}
+
func (c Caller) Info() CallerInfo {
return CallerInfo{
Package: c.Package(),
diff --git a/examples/core/api/main.go b/examples/core/api/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..88d14e452c3a5ff3fb472d92daa8f3386bdec922
--- /dev/null
+++ b/examples/core/api/main.go
@@ -0,0 +1,86 @@
+package main
+
+import (
+ "flag"
+ "math/rand"
+ "net/http"
+ "os"
+
+ "gitlab.com/uafrica/go-utils/api"
+ "gitlab.com/uafrica/go-utils/audit"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/examples/core/app"
+ "gitlab.com/uafrica/go-utils/examples/core/db"
+ "gitlab.com/uafrica/go-utils/logger"
+)
+
+func main() {
+ logger.SetGlobalLevel(logger.LevelDebug)
+ logger.SetGlobalFormat(logger.NewConsole())
+
+ localPort := flag.Int("port", 0, "Run with local HTTP server in this port (default: Run as lambda)")
+ flag.Parse()
+
+ api.New("uafrica-request-id", app.ApiRoutes()).
+ WithStarter("db", db.Connector("core")).
+ WithCheck("claims", claimsChecker{}).
+ WithCheck("maintenance", maint{}).
+ WithCheck("rate", rateLimiter{}).
+ WithCORS(cors{}).
+ WithAuditor(audit.File(os.Stdout)).
+ WithLocalPort(localPort, app.QueueRoutes()). //if nil will still run as lambda
+ Run()
+}
+
+type claimsChecker struct{}
+
+type Claims struct {
+ AccountID int64
+ UserID int64
+}
+
+func (claimsChecker) Check(ctx api.Context) (interface{}, error) {
+ //then extract auth claim and check against the db ...
+ claims := Claims{
+ UserID: 1,
+ AccountID: 13,
+ }
+
+ //set it in the API context (can be retrieved with api.Context.ClaimGet())
+ ctx.ClaimSet("AccountID", claims.AccountID)
+ ctx.ClaimSet("UserID", claims.UserID)
+
+ //return the struct (can be retrieved with service.Context.Get()/Value())
+ return claims, nil
+}
+
+type maint struct{}
+
+//for maintenance mode, put a message in environment variable MAINTENANCE_MODE
+//then than message will be displayed in the response. Clear the variable to
+//proceed normal operation
+func (m maint) Check(ctx api.Context) (interface{}, error) {
+ msg := os.Getenv("MAINTENANCE_MODE")
+ if msg != "" {
+ return nil, errors.HTTP(http.StatusTeapot, errors.Errorf("maintenance mode"), "maintenance mode")
+ }
+ return nil, nil //not maint mode
+}
+
+type rateLimiter struct{}
+
+func (r rateLimiter) Check(ctx api.Context) (interface{}, error) {
+ if rand.Intn(10) < 2 {
+ return nil, errors.Errorf("rate limited")
+ }
+ return nil, nil //not limited
+}
+
+type cors struct{}
+
+func (cors) CORS() map[string]string {
+ return map[string]string{
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Headers": "Content-Type, Accept",
+ }
+}
diff --git a/examples/core/app/app.go b/examples/core/app/app.go
new file mode 100644
index 0000000000000000000000000000000000000000..a1de7c506ed58014c2269c5ff20aa7e29502b3be
--- /dev/null
+++ b/examples/core/app/app.go
@@ -0,0 +1,33 @@
+package app
+
+import (
+ "gitlab.com/uafrica/go-utils/cron"
+ "gitlab.com/uafrica/go-utils/examples/core/app/users"
+ "gitlab.com/uafrica/go-utils/examples/core/email"
+)
+
+func ApiRoutes() map[string]map[string]interface{} {
+ return map[string]map[string]interface{}{
+ "/users": {
+ "GET": users.Get,
+ "POST": users.Add,
+ "PUT": users.Upd,
+ "DELETE": users.Del,
+ },
+ "/crash": {
+ "GET": users.Crash,
+ },
+ }
+}
+
+func QueueRoutes() map[string]interface{} {
+ return map[string]interface{}{
+ "email": email.Notify,
+ }
+}
+
+func CronRoutes() map[string]func(cron.Context) error {
+ return map[string]func(cron.Context) error{
+ "janitor": users.Janitor,
+ }
+}
diff --git a/examples/core/app/users/users.go b/examples/core/app/users/users.go
new file mode 100644
index 0000000000000000000000000000000000000000..93e094a2aaf803f1c3809d6bb00b17d747458fc6
--- /dev/null
+++ b/examples/core/app/users/users.go
@@ -0,0 +1,157 @@
+package users
+
+import (
+ "fmt"
+ "net/http"
+ "sync"
+ "time"
+
+ "gitlab.com/uafrica/go-utils/api"
+ "gitlab.com/uafrica/go-utils/cron"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/examples/core/email"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/service"
+)
+
+type User struct {
+ ID int `json:"id"`
+ AccountID int `json:"account_id"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Email string `json:"email"`
+}
+
+func (u User) Validate() error {
+ logger.Debugf("Validating (%T)%+v", u, u)
+ if u.ID != 0 {
+ return errors.Errorf("id may not be specified for new user")
+ }
+ if u.AccountID == 0 {
+ return errors.Errorf("missing account_id (from claims)")
+ }
+ if u.Email == "" {
+ return errors.Errorf("missing email")
+ }
+ if u.Username == "" {
+ return errors.Errorf("missing username")
+ }
+ if u.Password == "" {
+ return errors.Errorf("missing password")
+ }
+ return nil
+}
+
+//a simple in memory list of users for this example
+var (
+ usersMutex sync.Mutex
+ users = map[int]User{} //index on user.ID
+ nextUserID = 1
+)
+
+type getParams struct {
+ ID int `json:"id"`
+}
+
+func (p getParams) Validate() error {
+ if p.ID <= 0 {
+ return errors.Errorf("id>0 is required")
+ }
+ return nil
+}
+
+func Get(ctx api.Context, params getParams) (User, error) {
+ db := ctx.Value("db").(int)
+ ctx.Debugf("DB = %v", db)
+ ctx.Debugf("Claim: %+v", ctx.Claim())
+ ctx.Debugf("Params: %+v", params)
+ usersMutex.Lock()
+ defer usersMutex.Unlock()
+ if user, ok := users[params.ID]; ok {
+ return user, nil
+ }
+ return User{}, errors.HTTP(http.StatusNotFound, errors.Errorf("user.id=%d not found", params.ID), "")
+}
+
+type POSTUser struct {
+ User
+ U1 User `json:"u1"`
+ U2 *User `json:"u2"`
+}
+
+func Add(ctx api.Context, params noParams, newUser POSTUser) (User, error) {
+ db := ctx.Value("db").(int)
+ ctx.Debugf("DB = %v", db)
+ ctx.Debugf("Claim: %+v", ctx.Claim())
+
+ usersMutex.Lock()
+ defer usersMutex.Unlock()
+
+ //u1 and u2 not used - just to demonstrate how claims are populated in sub struct and sub struct ptrs
+ ctx.Debugf("u1: %+v", newUser.U1)
+ if newUser.U2 != nil {
+ ctx.Debugf("u2: %+v", newUser.U2)
+ }
+
+ //make sure user is unique for this account
+ for _, u := range users {
+ if u.AccountID == newUser.AccountID && u.Username == newUser.Username {
+ return User{}, errors.HTTP(http.StatusBadRequest, errors.Errorf("username \"%s\" already exists in this account", newUser.Username), "")
+ }
+ }
+
+ newUser.ID = nextUserID
+ nextUserID++
+ users[newUser.ID] = newUser.User
+
+ //send notification by email
+ email := email.Message{
+ To: []string{newUser.Email},
+ CC: nil,
+ BCC: nil,
+ From: "example@uafrica.com",
+ Subject: "Welcome User",
+ Body: "Your account has been created",
+ }
+ /*eventID*/ _, err := service.NewEvent(ctx, "notify").RequestID(ctx.RequestID()).Type("email").Delay(time.Second * 5).Params(map[string]string{}).Send(email)
+ if err != nil {
+ ctx.Errorf("failed to notify: %+v", err)
+ }
+ //ctx.Debugf("Notified: %v", eventID)
+ return newUser.User, nil
+}
+
+func Upd(ctx api.Context, params noParams, updUser User) (User, error) {
+ usersMutex.Lock()
+ defer usersMutex.Unlock()
+ existingUser, ok := users[updUser.ID]
+ if !ok {
+ return User{}, errors.HTTP(http.StatusNotFound, errors.Errorf("user.id=%d not found", updUser.ID), "")
+ }
+ if existingUser.Username != updUser.Username {
+ return User{}, errors.HTTP(http.StatusBadRequest, errors.Errorf("user.id=%d username=%s may not change", existingUser.ID, existingUser.Username), "")
+ }
+ users[updUser.ID] = updUser
+ return updUser, nil
+}
+
+func Del(ctx api.Context, params getParams) error {
+ usersMutex.Lock()
+ defer usersMutex.Unlock()
+ delete(users, params.ID)
+ return nil
+}
+
+type noParams struct{}
+
+func Crash(ctx api.Context, params noParams) error {
+ var d *int
+ fmt.Printf("%d", *d) //this should crash - causing the crash dumper to trigger
+ return nil
+}
+
+func Janitor(ctx cron.Context) error {
+ db := ctx.Value("db").(int)
+ ctx.Debugf("DB = %v", db)
+ return errors.Errorf("NYI")
+}
diff --git a/examples/core/cron/main.go b/examples/core/cron/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..f4f400ffcb1c19be1e908276c0ec8d373502942b
--- /dev/null
+++ b/examples/core/cron/main.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "math/rand"
+ "net/http"
+ "os"
+ "time"
+
+ "gitlab.com/uafrica/go-utils/api"
+ "gitlab.com/uafrica/go-utils/cron"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/examples/core/app"
+ "gitlab.com/uafrica/go-utils/examples/core/db"
+ "gitlab.com/uafrica/go-utils/logger"
+)
+
+func main() {
+ invokeArnPtr := flag.String("arn", "", "Invoke this ARN and terminate (default is to run as lambda)")
+ flag.Parse()
+
+ logger.SetGlobalLevel(logger.LevelDebug)
+ logger.SetGlobalFormat(logger.NewConsole())
+
+ cron.New(app.CronRoutes()).
+ WithStarter("db", db.Connector("core")).
+ //WithAuditor(audit{}).
+ Run(invokeArnPtr)
+}
+
+type maint struct{}
+
+//for maintenance mode, put a message in environment variable MAINTENANCE_MODE
+//then than message will be displayed in the response. Clear the variable to
+//proceed normal operation
+func (m maint) Check(ctx api.Context) (interface{}, error) {
+ msg := os.Getenv("MAINTENANCE_MODE")
+ if msg != "" {
+ return nil, errors.HTTP(http.StatusTeapot, errors.Errorf("maintenance mode"), "maintenance mode")
+ }
+ return nil, nil //not maint mode
+}
+
+type rateLimiter struct{}
+
+func (r rateLimiter) Check(ctx api.Context) (interface{}, error) {
+ if rand.Intn(10) < 2 {
+ return nil, errors.Errorf("rate limited")
+ }
+ return nil, nil //not limited
+}
+
+type cors struct{}
+
+func (cors) CORS() map[string]string {
+ return map[string]string{
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Headers": "Content-Type, Accept",
+ }
+}
+
+type audit struct{}
+
+func (a audit) Audit(startTime, endTime time.Time, values map[string]interface{}) {
+ fmt.Printf("AUDIT: %v %v %v ...\n", startTime, endTime, endTime.Sub(startTime))
+}
diff --git a/examples/core/db/database.go b/examples/core/db/database.go
new file mode 100644
index 0000000000000000000000000000000000000000..8021c42f85802f829e1640840ef425c057abcfe7
--- /dev/null
+++ b/examples/core/db/database.go
@@ -0,0 +1,38 @@
+package db
+
+import (
+ "math/rand"
+
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/service"
+)
+
+func Connector(dbName string) service.IStarter {
+ return &connector{
+ name: dbName,
+ conn: 0,
+ }
+}
+
+//connector implements service.IStarter
+type connector struct {
+ name string
+ conn int
+}
+
+//Start returns app specific data on success that can be retrieved by handlers
+func (c *connector) Start(ctx service.Context) (interface{}, error) {
+ //make reusable db connection
+ if c.conn < 2 {
+ c.conn = rand.Intn(10)
+ if c.conn < 2 {
+ return nil, errors.Errorf("failed to connect to db (just a random fake event - try again :-))")
+ }
+ ctx.Debugf("Connected to db(%s): %d", c.name, c.conn)
+ }
+
+ //return app data - here we only return db conn,
+ //which can be retrieved from ctx.Value("db"),
+ //because dbConnector{} is registered as "db"
+ return c.conn, nil
+}
diff --git a/examples/core/email/notify.go b/examples/core/email/notify.go
new file mode 100644
index 0000000000000000000000000000000000000000..206f2be9ee540314b853eeb68843298a2233467b
--- /dev/null
+++ b/examples/core/email/notify.go
@@ -0,0 +1,19 @@
+package email
+
+import (
+ "gitlab.com/uafrica/go-utils/queues"
+)
+
+type Message struct {
+ From string
+ To []string
+ CC []string
+ BCC []string
+ Subject string
+ Body string
+}
+
+func Notify(ctx queues.Context, msg Message) error {
+ ctx.Debugf("Pretending to send email: %+v", msg)
+ return nil
+}
diff --git a/examples/core/sqs/main.go b/examples/core/sqs/main.go
new file mode 100644
index 0000000000000000000000000000000000000000..7a019a7cf89b1d58996f51440fdd44cfa659c49f
--- /dev/null
+++ b/examples/core/sqs/main.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+ "flag"
+
+ "gitlab.com/uafrica/go-utils/config"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/examples/core/db"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/queues/sqs"
+)
+
+func main() {
+ reqFile := flag.String("req", "", "Request file to process for testing.")
+ flag.Parse()
+
+ sqsRoutes := map[string]interface{}{}
+
+ consumer := sqs.NewConsumer("uafrica-request-id", sqsRoutes).
+ WithStarter("db", db.Connector("core"))
+
+ if reqFile != nil && *reqFile != "" {
+ if err := config.LoadLocal(); err != nil {
+ panic(errors.Errorf("Failed to load local config: %+v", err))
+ }
+
+ if err := consumer.ProcessFile(*reqFile); err != nil {
+ panic(errors.Errorf("processing failed: %+v", err))
+ }
+ logger.Debugf("Stop after processing event from file.")
+ return
+ }
+
+ consumer.Run()
+}
diff --git a/go.mod b/go.mod
index e90b1e04ffa5949a3f40256c3df0f0053af4e0d0..4aae0d6879bbc18bcfa36f2d46f2ff900fd1a730 100644
--- a/go.mod
+++ b/go.mod
@@ -2,36 +2,39 @@ module gitlab.com/uafrica/go-utils
go 1.17
-require (
- github.com/go-pg/pg/v10 v10.10.5
- github.com/pkg/errors v0.9.1
- github.com/thoas/go-funk v0.9.1
- golang.org/x/text v0.3.6
-)
-
require (
github.com/aws/aws-lambda-go v1.26.0
+ github.com/aws/aws-sdk-go v1.40.50
+ github.com/aws/aws-secretsmanager-caching-go v1.1.0
+ github.com/cespare/xxhash/v2 v2.1.1 // indirect
+ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+ github.com/fatih/color v1.13.0
+ github.com/go-pg/pg/v10 v10.10.5
github.com/go-pg/zerochecker v0.2.0 // indirect
+ github.com/go-redis/redis/v8 v8.11.3
github.com/google/uuid v1.3.0
github.com/jinzhu/inflection v1.0.0 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+ github.com/mattn/go-colorable v0.1.9 // indirect
+ github.com/mattn/go-isatty v0.0.14 // indirect
+ github.com/pkg/errors v0.9.1
+ github.com/thoas/go-funk v0.9.1
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
- github.com/uptrace/bun v1.0.8
github.com/vmihailenco/bufpool v0.1.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+ golang.org/x/text v0.3.7
mellium.im/sasl v0.2.1 // indirect
)
require (
- github.com/aws/aws-sdk-go v1.40.50 // indirect
- github.com/cespare/xxhash/v2 v2.1.1 // indirect
- github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
- github.com/fatih/color v1.13.0 // indirect
- github.com/go-redis/redis/v8 v8.11.3 // indirect
- github.com/jmespath/go-jmespath v0.4.0 // indirect
- github.com/mattn/go-colorable v0.1.9 // indirect
- github.com/mattn/go-isatty v0.0.14 // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
+ github.com/r3labs/diff/v2 v2.14.0
+ github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
+ golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+ google.golang.org/appengine v1.6.6 // indirect
+ google.golang.org/protobuf v1.26.0 // indirect
)
diff --git a/go.sum b/go.sum
index 732bbf4426935a37d6b94894adb28668d189f9e1..7deed11eb9a7a373cf2531b8ffc04d044f9e48a0 100644
--- a/go.sum
+++ b/go.sum
@@ -2,8 +2,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
+github.com/aws/aws-sdk-go v1.19.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.40.50 h1:QP4NC9EZWBszbNo2UbG6bbObMtN35kCFb4h0r08q884=
github.com/aws/aws-sdk-go v1.40.50/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-secretsmanager-caching-go v1.1.0 h1:vcV94XGJ9KouXKYBTMqgrBw96Tae8JKLmoUZ5SbaXNo=
+github.com/aws/aws-secretsmanager-caching-go v1.1.0/go.mod h1:wahQpJP1dZKMqjGFAjGCqilHkTlN0zReGWocPLbXmxg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -32,6 +35,7 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -42,6 +46,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -55,8 +60,10 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -69,17 +76,16 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
@@ -88,6 +94,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/r3labs/diff/v2 v2.14.0 h1:VRI8lhKFP4miM+RlyKkdoT94u7RlFge2S+WAqDScV2Q=
+github.com/r3labs/diff/v2 v2.14.0/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -100,12 +108,11 @@ github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
-github.com/uptrace/bun v1.0.8 h1:uWrBnun83dfNGP//xdLAdGA3UGISQtpdSzjZHUNmZh0=
-github.com/uptrace/bun v1.0.8/go.mod h1:aL6D9vPw8DXaTQTwGrEPtUderBYXx7ShUmPfnxnqscw=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
-github.com/vmihailenco/msgpack/v5 v5.3.1 h1:0i85a4dsZh8mC//wmyyTEzidDLPQfQAxZIOLtafGbFY=
+github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
+github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc=
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
@@ -131,13 +138,14 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -159,18 +167,16 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7 h1:iGu644GcxtEcrInvDsQRCwJjtCIOlT2V7IRt6ah2Whw=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -180,11 +186,12 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
@@ -201,6 +208,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -212,11 +220,11 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/logger/context.go b/logger/context.go
index 9f709dd5dec4f850d6834e3fb3d8a8a6376155e3..9e5195975b6b8afe3e507222853bec3d057a2d17 100644
--- a/logger/context.go
+++ b/logger/context.go
@@ -5,88 +5,40 @@ import (
)
func GetContextLogger() Logger {
- return logger
+ return globalLogger
}
-// func ClearInfo() {
-// InitLogs(nil, nil)
-// }
-
-// func InitLogs(requestID *string, request *events.APIGatewayProxyRequest) {
-// logger = Logger{
-// currentRequestID: requestID,
-// apiRequest: request,
-// level: LevelInfo,
-// writer: os.Stderr,
-// data: map[string]interface{}{
-// "environment": getEnvironment(),
-// },
-// }
-// if requestID != nil {
-// logger.data["request_id"] = *requestID
-// }
-
-// if val, exists := os.LookupEnv("DEBUGGING"); exists && strings.ToLower(val) == "true" {
-// logger.level = LevelDebug
-// }
-// }
-
-// func getEnvironment() string {
-// environment := os.Getenv("ENVIRONMENT")
-// if environment == "" {
-// environment = "prod"
-// }
-// return environment
-// }
-
func LogMessageWithFields(fields map[string]interface{}, message interface{}) {
- logger.WithFields(fields).log(LevelInfo, 1, fmt.Sprintf("%v", message))
+ globalLogger.WithFields(fields).log(LevelInfo, 1, fmt.Sprintf("%v", message))
}
func LogMessage(format string, a ...interface{}) {
- logger.log(LevelInfo, 1, fmt.Sprintf(format, a...))
+ globalLogger.log(LevelInfo, 1, fmt.Sprintf(format, a...))
}
func LogError(fields map[string]interface{}, err error) {
// sendRaygunError(fields, err)
- logger.WithFields(fields).log(LevelError, 1, fmt.Sprintf("%+v", err))
+ globalLogger.WithFields(fields).log(LevelError, 1, fmt.Sprintf("%+v", err))
}
func LogErrorMessage(message interface{}, err error) {
if err != nil || message != nil {
- logger.WithFields(map[string]interface{}{
+ globalLogger.WithFields(map[string]interface{}{
"error": fmt.Sprintf("%+v", err),
}).log(LevelError, 1, fmt.Sprintf("%v", message))
}
}
func LogWarningMessage(format string, a ...interface{}) {
- logger.log(LevelWarn, 1, fmt.Sprintf(format, a...))
+ globalLogger.log(LevelWarn, 1, fmt.Sprintf(format, a...))
}
func LogWarning(fields map[string]interface{}, err error) {
- logger.WithFields(fields).log(LevelWarn, 1, fmt.Sprintf("%+v", err))
+ globalLogger.WithFields(fields).log(LevelWarn, 1, fmt.Sprintf("%+v", err))
}
func SQLDebugInfo(sql string) {
- logger.WithFields(map[string]interface{}{
+ globalLogger.WithFields(map[string]interface{}{
"sql": sql,
}).log(LevelInfo, 1, "SQL")
}
-
-// func LogRequestInfo(req events.APIGatewayProxyRequest) {
-// fields := map[string]interface{}{
-// "http_method": req.HTTPMethod,
-// "path": req.Path,
-// "api_gateway_request_id": req.RequestContext.RequestID,
-// "user_cognito_auth_provider": req.RequestContext.Identity.CognitoAuthenticationProvider,
-// "user_arn": req.RequestContext.Identity.UserArn,
-// }
-// logger.WithFields(fields).log(LevelInfo, 1, "Request Info start")
-// }
-
-// func LogSQSEvent(event events.SQSEvent) {
-// logger.WithFields(map[string]interface{}{
-// "records": event.Records,
-// }).log(LevelInfo, 1, "SQS event start")
-// }
diff --git a/logger/format.go b/logger/format.go
new file mode 100644
index 0000000000000000000000000000000000000000..e960f8117f20e7c624abf337fe724504fa7ec252
--- /dev/null
+++ b/logger/format.go
@@ -0,0 +1,161 @@
+package logger
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+
+ "github.com/fatih/color"
+)
+
+var nextFormatID = 1
+
+type IFormatter interface {
+ Format(Entry) []byte
+ NextColor() IFormatter
+ Color() string
+}
+
+func NewJSON() IFormatter {
+ return formatterJSON{}
+}
+
+type formatterJSON struct{}
+
+func (f formatterJSON) Format(entry Entry) []byte {
+ jsonEntry, err := json.Marshal(entry)
+ if err != nil {
+ return []byte(fmt.Sprintf("failed to marshal entry: %v: %+v\n", err, entry))
+ }
+ return append(jsonEntry, []byte("\n")...)
+}
+
+func (f formatterJSON) NextColor() IFormatter {
+ return f //do not select colors for JSON (only used in console)
+}
+
+func (f formatterJSON) Color() string {
+ return "default"
+}
+
+func NewConsole() IFormatter {
+ nextFormatID++
+ return formatterConsole{
+ id: nextFormatID,
+ fg: 1, //color.FgWhite,
+ bg: 0, //color.BgBlack,
+ }
+}
+
+type formatterConsole struct {
+ id int
+ fg int //color.Attribute
+ bg int //color.Attribute
+}
+
+func (f formatterConsole) Format(entry Entry) []byte {
+ source := fmt.Sprintf("%s/%s:%d", entry.Caller.Package, entry.Caller.File, entry.Caller.Line)
+ if len(source) > 40 {
+ source = source[len(source)-40:]
+ }
+
+ buffer := bytes.NewBuffer(nil)
+
+ red := color.New(color.FgRed).FprintfFunc()
+ magenta := color.New(color.FgMagenta).FprintfFunc()
+ yellow := color.New(color.FgYellow).FprintfFunc()
+ //blue := color.New(color.FgBlue).FprintfFunc()
+ green := color.New(color.FgGreen).FprintfFunc()
+ cyan := color.New(color.FgCyan).FprintfFunc()
+
+ cyan(buffer, entry.Timestamp.Format("2006-01-02 15:04:05"))
+
+ levelString := fmt.Sprintf(" %5.5s", entry.Level)
+ switch entry.Level {
+ case LevelFatal:
+ red(buffer, levelString)
+ case LevelError:
+ red(buffer, levelString)
+ case LevelWarn:
+ magenta(buffer, levelString)
+ case LevelInfo:
+ yellow(buffer, levelString)
+ case LevelDebug:
+ green(buffer, levelString)
+ }
+ cyan(buffer, fmt.Sprintf(" %-40.40s| ", source))
+
+ base := color.New(fgColors[colorNames[f.fg]], bgColors[colorNames[f.bg]]).FprintfFunc()
+ base(buffer, entry.Message) //buffer.Write([]byte(entry.Message))
+
+ if len(entry.Data) > 0 {
+ jsonData, _ := json.Marshal(entry.Data)
+ green(buffer, " "+string(jsonData))
+ }
+ buffer.WriteString("\n")
+ return buffer.Bytes()
+}
+
+func (f formatterConsole) WithForeground(fg color.Attribute) IFormatter {
+ // f.fg = fg
+ return f
+}
+
+func (f formatterConsole) WithBackground(bg color.Attribute) IFormatter {
+ // f.bg = bg
+ return f
+}
+
+func (f formatterConsole) Color() string {
+ return colorNames[f.fg] + " on " + colorNames[f.bg]
+}
+
+var (
+ colorNames = []string{"black", "white", "red", "green", "yellow", "blue", "magenta", "cyan"}
+ fgColors = map[string]color.Attribute{
+ "black": color.FgBlack,
+ "white": color.FgWhite,
+ "red": color.FgRed,
+ "green": color.FgGreen,
+ "yellow": color.FgYellow,
+ "blue": color.FgBlue,
+ "magenta": color.FgMagenta,
+ "cyan": color.FgCyan,
+ }
+ bgColors = map[string]color.Attribute{
+ "black": color.BgBlack,
+ "white": color.BgWhite,
+ "red": color.BgRed,
+ "green": color.BgGreen,
+ "yellow": color.BgYellow,
+ "blue": color.BgBlue,
+ "magenta": color.BgMagenta,
+ "cyan": color.BgCyan,
+ }
+ nextFg = 1
+ nextBg = 0
+)
+
+func (f formatterConsole) NextColor() IFormatter {
+ for {
+ nextFg++
+ if nextFg >= len(fgColors) {
+ nextFg = 0
+ // nextBg++
+ // if nextBg >= len(bgColors) {
+ // nextBg++
+ // }
+ }
+ if nextFg != nextBg {
+ break
+ }
+ }
+ nextFormatID++
+ f = formatterConsole{
+ id: nextFormatID,
+ fg: nextFg,
+ bg: nextBg,
+ }
+ //globalLogger.Debugf("NextColor return id=%d Color %s on %s", f.id, colorNames[f.fg], colorNames[f.bg])
+ return f
+}
diff --git a/logger/global.go b/logger/global.go
index f65afc41deb7a35be4195ce740d381874fc64b99..c04253667b5c9dff7402c2d2a5e853fce8cd5478 100644
--- a/logger/global.go
+++ b/logger/global.go
@@ -4,69 +4,75 @@ import (
"fmt"
"os"
- "github.com/fatih/color"
-
"gitlab.com/uafrica/go-utils/errors"
)
-var logger Logger
+var globalLogger logger
func init() {
- logger = Logger{
- level: LevelDebug,
- writer: os.Stderr,
- data: map[string]interface{}{},
- fg: color.FgWhite,
- bg: color.BgBlack,
+ globalLogger = logger{
+ level: LevelInfo, //default for production
+ writer: os.Stderr,
+ data: map[string]interface{}{},
+ IFormatter: formatterJSON{}, //default to JSON format for production
+ }
+}
+
+func SetGlobalLevel(level Level) {
+ globalLogger.level = level
+}
+
+func SetGlobalFormat(f IFormatter) {
+ if f != nil {
+ globalLogger.IFormatter = f
}
- // InitLogs(nil, nil)
}
func New() Logger {
- return logger.WithFields(nil)
+ return globalLogger.WithFields(nil)
}
//shortcut functions to use current logger
//this should only be used outside of a request context
//or anywhere if you have a single threaded process
func Fatalf(format string, args ...interface{}) {
- logger.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprintf(format, args...))
+ globalLogger.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprintf(format, args...))
os.Exit(1)
}
func Fatal(args ...interface{}) {
- logger.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprint(args...))
+ globalLogger.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprint(args...))
os.Exit(1)
}
func Errorf(format string, args ...interface{}) {
- logger.log(LevelError, 1, fmt.Sprintf(format, args...))
+ globalLogger.log(LevelError, 1, fmt.Sprintf(format, args...))
}
func Error(args ...interface{}) {
- logger.log(LevelError, 1, fmt.Sprint(args...))
+ globalLogger.log(LevelError, 1, fmt.Sprint(args...))
}
func Warnf(format string, args ...interface{}) {
- logger.log(LevelWarn, 1, fmt.Sprintf(format, args...))
+ globalLogger.log(LevelWarn, 1, fmt.Sprintf(format, args...))
}
func Warn(args ...interface{}) {
- logger.log(LevelWarn, 1, fmt.Sprint(args...))
+ globalLogger.log(LevelWarn, 1, fmt.Sprint(args...))
}
func Infof(format string, args ...interface{}) {
- logger.log(LevelInfo, 1, fmt.Sprintf(format, args...))
+ globalLogger.log(LevelInfo, 1, fmt.Sprintf(format, args...))
}
func Info(args ...interface{}) {
- logger.log(LevelInfo, 1, fmt.Sprint(args...))
+ globalLogger.log(LevelInfo, 1, fmt.Sprint(args...))
}
func Debugf(format string, args ...interface{}) {
- logger.log(LevelDebug, 1, fmt.Sprintf(format, args...))
+ globalLogger.log(LevelDebug, 1, fmt.Sprintf(format, args...))
}
func Debug(args ...interface{}) {
- logger.log(LevelDebug, 1, fmt.Sprint(args...))
+ globalLogger.log(LevelDebug, 1, fmt.Sprint(args...))
}
diff --git a/logger/logger.go b/logger/logger.go
index 8347fc56399c102740390e07446c965415e998fc..736846d0ec5117c1f6090c2bf3b2061c87d5a49f 100644
--- a/logger/logger.go
+++ b/logger/logger.go
@@ -1,18 +1,15 @@
package logger
import (
- "bytes"
- "encoding/json"
"fmt"
"io"
"strings"
"time"
- "github.com/fatih/color"
"gitlab.com/uafrica/go-utils/errors"
)
-type ILogger interface {
+type Logger interface {
Fatalf(format string, args ...interface{})
Fatal(args ...interface{})
Errorf(format string, args ...interface{})
@@ -23,88 +20,23 @@ type ILogger interface {
Info(args ...interface{})
Debugf(format string, args ...interface{})
Debug(args ...interface{})
+
+ WithFields(data map[string]interface{}) logger
}
-type Logger struct {
- //apiRequest *events.APIGatewayProxyRequest
- //currentRequestID *string
+type logger struct {
level Level
writer io.Writer
data map[string]interface{}
-
- fg color.Attribute
- bg color.Attribute
-}
-
-func (l Logger) WithForeground(fg color.Attribute) Logger {
- l.fg = fg
- return l
-}
-
-func (l Logger) WithBackground(bg color.Attribute) Logger {
- l.bg = bg
- return l
-}
-
-type textColor struct {
- fg color.Attribute
- bg color.Attribute
-}
-
-var (
- fgColors = []color.Attribute{
- color.FgWhite,
- color.FgRed,
- color.FgGreen,
- color.FgYellow,
- color.FgBlue,
- color.FgMagenta,
- color.FgCyan,
- color.FgBlack,
- }
- bgColors = []color.Attribute{
- color.BgBlack,
- color.BgWhite,
- color.BgRed,
- color.BgGreen,
- color.BgYellow,
- color.BgBlue,
- color.BgMagenta,
- color.BgCyan,
- }
- nextFg = 0
- nextBg = 0
-)
-
-func (l Logger) NextColor() Logger {
- l.fg = 0
- l.bg = 0
- for l.fg == l.bg {
- l.bg = bgColors[nextBg]
- l.fg = fgColors[nextFg]
- incColor()
- }
- return l
+ IFormatter
}
-func incColor() {
- nextFg++
- if nextFg >= len(fgColors) {
- nextFg = 0
- nextBg++
- if nextBg >= len(bgColors) {
- nextBg++
- }
- }
-}
-
-func (l Logger) WithFields(data map[string]interface{}) Logger {
- newLogger := Logger{
- level: l.level,
- writer: l.writer,
- data: map[string]interface{}{},
- fg: l.fg,
- bg: l.bg,
+func (l logger) WithFields(data map[string]interface{}) logger {
+ newLogger := logger{
+ level: l.level,
+ writer: l.writer,
+ data: map[string]interface{}{},
+ IFormatter: l.IFormatter,
}
for n, v := range l.data {
newLogger.data[n] = v
@@ -115,47 +47,47 @@ func (l Logger) WithFields(data map[string]interface{}) Logger {
return newLogger
}
-func (l Logger) Fatalf(format string, args ...interface{}) {
+func (l logger) Fatalf(format string, args ...interface{}) {
l.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprintf(format, args...))
}
-func (l Logger) Fatal(args ...interface{}) {
+func (l logger) Fatal(args ...interface{}) {
l.WithFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprint(args...))
}
-func (l Logger) Errorf(format string, args ...interface{}) {
+func (l logger) Errorf(format string, args ...interface{}) {
l.log(LevelError, 1, fmt.Sprintf(format, args...))
}
-func (l Logger) Error(args ...interface{}) {
+func (l logger) Error(args ...interface{}) {
l.log(LevelError, 1, fmt.Sprint(args...))
}
-func (l Logger) Warnf(format string, args ...interface{}) {
+func (l logger) Warnf(format string, args ...interface{}) {
l.log(LevelWarn, 1, fmt.Sprintf(format, args...))
}
-func (l Logger) Warn(args ...interface{}) {
+func (l logger) Warn(args ...interface{}) {
l.log(LevelWarn, 1, fmt.Sprint(args...))
}
-func (l Logger) Infof(format string, args ...interface{}) {
+func (l logger) Infof(format string, args ...interface{}) {
l.log(LevelInfo, 1, fmt.Sprintf(format, args...))
}
-func (l Logger) Info(args ...interface{}) {
+func (l logger) Info(args ...interface{}) {
l.log(LevelInfo, 1, fmt.Sprint(args...))
}
-func (l Logger) Debugf(format string, args ...interface{}) {
+func (l logger) Debugf(format string, args ...interface{}) {
l.log(LevelDebug, 1, fmt.Sprintf(format, args...))
}
-func (l Logger) Debug(args ...interface{}) {
+func (l logger) Debug(args ...interface{}) {
l.log(LevelDebug, 1, fmt.Sprint(args...))
}
-func (l Logger) log(level Level, skip int, msg string) {
+func (l logger) log(level Level, skip int, msg string) {
if level <= l.level && l.writer != nil {
entry := Entry{
Timestamp: time.Now(),
@@ -164,50 +96,7 @@ func (l Logger) log(level Level, skip int, msg string) {
Data: l.data,
Message: strings.ReplaceAll(msg, "\n", ";"),
}
-
- // jsonEntry, err := json.Marshal(entry)
- // if err != nil {
- // l.writer.Write([]byte(fmt.Sprintf("failed to marshal entry: %v: %+v\n", err, entry)))
- // }
- // l.writer.Write(append(jsonEntry, []byte("\n")...))
-
- source := fmt.Sprintf("%s/%s:%d", entry.Caller.Package, entry.Caller.File, entry.Caller.Line)
- if len(source) > 40 {
- source = source[len(source)-40:]
- }
-
- buffer := bytes.NewBuffer(nil)
- red := color.New(color.FgRed).FprintfFunc()
- magenta := color.New(color.FgMagenta).FprintfFunc()
- yellow := color.New(color.FgYellow).FprintfFunc()
- blue := color.New(color.FgBlue).FprintfFunc()
- green := color.New(color.FgGreen).FprintfFunc()
- cyan := color.New(color.FgCyan).FprintfFunc()
- blue(buffer, entry.Timestamp.Format("2006-01-02 15:04:05"))
- levelString := fmt.Sprintf(" %5.5s", entry.Level)
- switch entry.Level {
- case LevelFatal:
- red(buffer, levelString)
- case LevelError:
- red(buffer, levelString)
- case LevelWarn:
- magenta(buffer, levelString)
- case LevelInfo:
- yellow(buffer, levelString)
- case LevelDebug:
- green(buffer, levelString)
- }
- cyan(buffer, fmt.Sprintf(" %-40.40s| ", source))
-
- base := color.New(l.fg, l.bg).FprintfFunc()
- base(buffer, entry.Message) //buffer.Write([]byte(entry.Message))
-
- if len(entry.Data) > 0 {
- jsonData, _ := json.Marshal(entry.Data)
- green(buffer, " "+string(jsonData))
- }
- buffer.WriteString("\n")
- l.writer.Write(buffer.Bytes())
+ l.writer.Write(l.Format(entry))
}
}
@@ -218,85 +107,3 @@ type Entry struct {
Data map[string]interface{} `json:"data"`
Message string `json:"message"`
}
-
-// func sendRaygunError(fields map[string]interface{}, errToSend error) {
-// if os.Getenv("DEBUGGING") == "true" {
-// // Don't log raygun errors on debug
-// return
-// }
-
-// raygun, err := RaygunReporter()
-// if err != nil || raygun == nil {
-// logger.Errorf("Unable to create Raygun client: " + err.Error())
-// return
-// }
-
-// env := getEnvironment()
-// tags := []string{env}
-// //todo: raygun.Version(globals.BuildVersion)
-// //todo: tags = append(tags, globals.BuildVersion)
-
-// if logger.apiRequest != nil {
-// methodAndPath := logger.apiRequest.HTTPMethod + ": " + logger.apiRequest.Path
-// tags = append(tags, methodAndPath)
-// fields["body"] = logger.apiRequest.Body
-// fields["query"] = logger.apiRequest.QueryStringParameters
-// fields["identity"] = logger.apiRequest.RequestContext.Identity
-// }
-
-// raygun.Tags(tags)
-// if logger.currentRequestID != nil {
-// fields["request_id"] = logger.currentRequestID
-// }
-
-// fields["env"] = env
-// raygun.CustomData(fields)
-// raygun.Request(fakeHttpRequest())
-
-// if errToSend == nil {
-// errToSend = errors.New("")
-// }
-// err = raygun.SendError(errToSend)
-// if err != nil {
-// logger.Errorf("Failed to send raygun error: " + err.Error())
-// }
-// }
-
-// func fakeHttpRequest() *http.Request {
-// if logger.apiRequest == nil {
-// return nil
-// }
-
-// requestURL := url.URL{
-// Path: logger.apiRequest.Path,
-// Host: logger.apiRequest.Headers["Host"],
-// }
-// request := http.Request{
-// Method: logger.apiRequest.HTTPMethod,
-// URL: &requestURL,
-// Header: logger.apiRequest.MultiValueHeaders,
-// }
-// return &request
-// }
-
-// func RaygunReporter() (*raygun4go.Client, error) {
-// key := "AwdpHhwOF1lTT6AppyEHA"
-
-// // TODO Raygun per environment
-// //env := getEnvironment()
-// //if env == "dev" {
-// // key = "q98iTyE5CcF7raZsIiLA"
-// //} else if env == "stage" {
-// // key = "TA54mzcv9cBWBLmfwIQMDg"
-// //} else if env == "prod" {
-// // key = "BXdraqiPKBXImqP4siK1w"
-// //}
-
-// raygun, err := raygun4go.New("uAfrica V3", key)
-// if err != nil || raygun == nil {
-// logger.Errorf("Unable to create Raygun client:" + err.Error())
-// return nil, err
-// }
-
-// return raygun, nil
-// }
diff --git a/logger/logs_test.go b/logger/logs_test.go
index 8ebb965bd9b07f0309aa027cbc73caed5d74ed7c..768865dfe0574bedac6160528ab6ccf46cbda018 100644
--- a/logger/logs_test.go
+++ b/logger/logs_test.go
@@ -43,4 +43,15 @@ func TestColor(t *testing.T) {
// Mix up with multiple attributes
success := color.New(color.Bold, color.FgGreen).FprintlnFunc()
success(os.Stdout, " don't forget this...")
+
+ logger.SetGlobalFormat(logger.NewConsole())
+ logger.SetGlobalLevel(logger.LevelDebug)
+ logger.Debugf("Main logger")
+ l := logger.New()
+ l.Debugf("Logger 1")
+ l = logger.New()
+ l.Debugf("Logger 2")
+ // l = logger.New()
+ // l.IFormatter = l.IFormatter.NextColor()
+ // l.Debugf("Logger 3")
}
diff --git a/logger/stack.go b/logger/stack.go
new file mode 100644
index 0000000000000000000000000000000000000000..0a178d000125578a2d882b4a20a2a67cd101c970
--- /dev/null
+++ b/logger/stack.go
@@ -0,0 +1,91 @@
+package logger
+
+import (
+ "bufio"
+ "runtime/debug"
+ "strconv"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+type Stack struct {
+ Routine int64 //go routine nr that crashed...
+ Callers []errors.CallerInfo
+}
+
+func CallStack() Stack {
+ stack := Stack{
+ Callers: []errors.CallerInfo{},
+ }
+
+ //get the call stack
+ s := bufio.NewScanner(strings.NewReader(string(debug.Stack())))
+ //expect stack to look like this:
+ // "goroutine 14 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:24 +0x88\ngitlab.com/uafrica/go-utils/api.Api.Handler.func3.1(0x1400009ff60)\n\t/Users/jansemmelink/uafrica/go-utils/api/lambda.go:210 +0x48\npanic({0x100780d20, 0x100b98a50})\n\t/usr/local/go/src/runtime/panic.go:1038 +0x21c\ngitlab.com/uafrica/go-utils/examples/core/api/users.Crash({{{0x1008237e0, 0x1400040af30}, {0x4, {0x1008186e0, 0x14000010020}, 0x1400040af90, 0x25, 0x2f}, {0xc04f2cb513fbdc28, 0x103ccb5b4c, ...}}, ...}, ...)\n\t/Users/jansemmelink/uafrica/go-utils/examples/core/api/users/users.go:115 +0x20\nreflect.Value.call({0x10076bba0, 0x10080cdc8, 0x13}, {0x10059bb1c, 0x4}, {0x140000a0730, 0x2, 0x2})\n\t/usr/local/go/src/reflect/value.go:543 +0x584\nreflect.Value.Call({0x10076bba0, 0x10080cdc8, 0x13}, {0x140000a0730, 0x2, 0x2})\n\t/usr/local/go/src/reflect/value.go:339 +0x8c\ngitlab.com/uafrica/go-utils/api.Api.Handler.func3({{0x10082f730, 0x100780960}, {0x0, 0x0}, {0x0, 0x0}, {0x10076bba0, 0x10080cdc8, 0x13}}, {0x140000a0730, ...})\n\t/Users/jansemmelink/uafrica/go-utils/api/lambda.go:214 +0x84\ngitlab.com/uafrica/go-utils/api.Api.Handler({{0x10082b3f0, 0x140003b6480}, {0x10059b80a, 0x3}, {0x140003b6360}, {0x1005a3606, 0x12}, 0x10080cf98, {0x1008190e0, 0x100bdd308}, ...}, ...)\n\t/Users/jansemmelink/uafrica/go-utils/api/lambda.go:216 +0x1238\ngitlab.com/uafrica/go-utils/api.Api.ServeHTTP({{0x10082b3f0, 0x140003b6480}, {0x10059b80a, 0x3}, {0x140003b6360}, {0x1005a3606, 0x12}, 0x10080cf98, {0x1008190e0, 0x100bdd308}, ...}, ...)\n\t/Users/jansemmelink/uafrica/go-utils/api/local.go:81 +0x6ac\nnet/http.serverHandler.ServeHTTP({0x1400015a620}, {0x1008200e8, 0x1400015aa80}, 0x140003c6900)\n\t/usr/local/go/src/net/http/server.go:2878 +0x444\nnet/http.(*conn).serve(0x14000279220, {0x1008237e0, 0x140003b6780})\n\t/usr/local/go/src/net/http/server.go:1929 +0xb6c\ncreated by net/http.(*Server).Serve\n\t/usr/local/go/src/net/http/server.go:3033 +0x4b8\n"
+ //i.e. multiple lines ending with "\n" each, e.g.:
+ //------------------------------------------------------------------------------------------------------------
+ // goroutine 37 [running]:
+ // runtime/debug.Stack()
+ // /usr/local/go/src/runtime/debug/stack.go:24 +0x88
+ // gitlab.com/uafrica/go-utils/api.Api.Handler.func3.1(0x1400042cb00, 0x14000095f68)
+ // /Users/jansemmelink/uafrica/go-utils/api/lambda.go:216 +0x50
+ // panic({0x100c08d20, 0x101020a70})
+ // /usr/local/go/src/runtime/panic.go:1038 +0x21c
+ // gitlab.com/uafrica/go-utils/examples/core/api/users.Crash({{{0x100cab7c0, 0x140004843c0}, {0x4, {0x100ca06c0, 0x14000138010}, 0x14000484420, 0x20, 0x28}, {0xc04f2d6c3023b330, 0x227861e6a, ...}}, ...}, ...)
+ // /Users/jansemmelink/uafrica/go-utils/examples/core/api/users/users.go:115 +0x20
+ // ...
+ //------------------------------------------------------------------------------------------------------------
+
+ //get go routine nr from first line: "gorouting <nr> [running]:"
+ if s.Scan() {
+ p := strings.SplitN(s.Text(), " ", 3)
+ if len(p) >= 2 && p[0] == "goroutine" {
+ routineNr, err := strconv.ParseInt(p[1], 10, 64)
+ if err == nil {
+ stack.Routine = routineNr
+ }
+ }
+ }
+
+ //next expect line pairs for each level of the stack
+ for {
+ //read first line in this pair, expecting <package>.<funcName>(<args>)
+ if !s.Scan() {
+ break
+ }
+ line1 := s.Text()
+
+ if !s.Scan() {
+ break
+ }
+ line2 := s.Text()
+
+ //fmt.Printf(" STACK LINE: %s %s\n", line1, line2)
+
+ //split line 1 on any bracket or comma to get "<package>.<funcName>"["<arg>" ...]
+ //func may have multiple '.', so do not split on that yet!
+ line1Fields := strings.FieldsFunc(line1, func(c rune) bool { return strings.Contains("(), ", string(c)) })
+
+ //split line 2 <file>:<line> +0x##...
+ line2Fields := strings.FieldsFunc(line2, func(c rune) bool { return strings.Contains(": ", string(c)) })
+ lineNr, _ := strconv.ParseInt(line2Fields[1], 10, 64)
+ caller := errors.NewCaller(line1Fields[0], line2Fields[0], int(lineNr))
+
+ //skip first levels that refer to capturing the stack
+ ci := caller.Info()
+ if len(stack.Callers) == 0 {
+ if ci.Package == "runtime/debug" ||
+ ci.Package == "gitlab.com/uafrica/go-utils/logger" ||
+ ci.Package == "gitlab.com/uafrica/go-utils/errors" ||
+ ci.Package == "" {
+ continue
+ }
+ if _, err := strconv.ParseInt(ci.Function, 10, 64); err == nil {
+ continue //typical defer function without a name used to catch the crash
+ }
+ }
+ stack.Callers = append(stack.Callers, caller.Info())
+ }
+ return stack
+}
diff --git a/queues/audit.go b/queues/audit.go
new file mode 100644
index 0000000000000000000000000000000000000000..16d50098946c8aecc76aa2b26bffb9011e13fdda
--- /dev/null
+++ b/queues/audit.go
@@ -0,0 +1,55 @@
+package queues
+
+import (
+ "time"
+
+ "gitlab.com/uafrica/go-utils/audit"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/service"
+)
+
+//create auditor that push to a queue using the specified producer
+func Auditor(queueName string, messageType string, producer service.ProducerLogger) audit.Auditor {
+ if producer == nil {
+ panic(errors.Errorf("cannot create auditor with producer=nil"))
+ }
+ if queueName == "" {
+ queueName = "AUDIT"
+ }
+ if messageType == "" {
+ messageType = "audit"
+ }
+ return auditor{
+ producer: producer,
+ queueName: queueName,
+ messageType: messageType,
+ }
+}
+
+type auditor struct {
+ producer service.ProducerLogger
+ queueName string
+ messageType string
+}
+
+func (a auditor) WriteEvent(requestID string, event audit.Event) error {
+ _, err := service.NewEvent(a.producer, a.queueName).
+ RequestID(requestID).
+ Type(a.messageType).
+ Send(event)
+ if err != nil {
+ return errors.Wrapf(err, "failed to write audit event")
+ }
+ return nil
+}
+
+func (a auditor) WriteValues(startTime, endTime time.Time, requestID string, values map[string]interface{}) error {
+ _, err := service.NewEvent(a.producer, a.queueName).
+ RequestID(requestID).
+ Type(a.messageType).
+ Send(values)
+ if err != nil {
+ return errors.Wrapf(err, "failed to write audit values")
+ }
+ return nil
+}
diff --git a/queues/check.go b/queues/check.go
new file mode 100644
index 0000000000000000000000000000000000000000..f0148eef81b12737ba3b85d1082f0ecf99711956
--- /dev/null
+++ b/queues/check.go
@@ -0,0 +1,5 @@
+package queues
+
+type ICheck interface {
+ Check(Context) (interface{}, error)
+}
diff --git a/queues/consumer.go b/queues/consumer.go
index 7549b85ea66e253c1a1a4665181b2ed773468431..a476ddc1a994ea154851c499b652c753fb148e5e 100644
--- a/queues/consumer.go
+++ b/queues/consumer.go
@@ -2,8 +2,10 @@ package queues
import "gitlab.com/uafrica/go-utils/service"
-type IConsumer interface {
- WithDb(dbConn service.IDatabaseConnector) IConsumer
+//IConsumer is the interface implemented by both mem and sqs consumer
+type Consumer interface {
+ WithStarter(name string, starter service.IStarter) Consumer
+ WithErrorReporter(reporter service.IErrorReporter) Consumer
Run()
ProcessFile(filename string) error
}
diff --git a/queues/context.go b/queues/context.go
index 335bb870f959d97d9cb2180b3377b8bf77d1a29c..c3321fecabf9238b4dd28da4616ede30bbb3dc99 100644
--- a/queues/context.go
+++ b/queues/context.go
@@ -4,34 +4,52 @@ import (
"context"
"encoding/json"
"reflect"
- "time"
- "github.com/uptrace/bun"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
"gitlab.com/uafrica/go-utils/service"
)
-type IContext interface {
- context.Context
- logger.ILogger
- IProducer
- StartTime() time.Time
- MillisecondsSinceStart() int64
- //DB() *bun.DB
+//Context within a consumer to process an event
+type Context interface {
+ service.Context
+ Event() service.Event //the event start started this context in the consumer
+ GetRecord(recordType reflect.Type) (interface{}, error) //extract struct value from event data
}
-type Context struct {
+var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
+
+type queuesContext struct {
service.Context
- IProducer //todo: would be nice to have a method to requeue same event with delay and incrementing attempt nr
- Event Event
- RequestID string
- DB *bun.DB
+ event service.Event
+}
+
+func NewContext(service service.Service, event service.Event) (Context, error) {
+ baseCtx := context.Background()
+ serviceContext, err := service.NewContext(baseCtx, event.RequestIDValue, map[string]interface{}{
+ "message_type": event.TypeName,
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create service context")
+ }
+
+ ctx := queuesContext{
+ Context: serviceContext,
+ event: event,
+ }
+ return ctx, nil
+}
+
+func (ctx queuesContext) Event() service.Event {
+ return ctx.event
+}
+
+func (ctx queuesContext) RequestID() string {
+ return ctx.event.RequestIDValue
}
-func (ctx Context) GetRecord(recordType reflect.Type) (interface{}, error) {
+func (ctx queuesContext) GetRecord(recordType reflect.Type) (interface{}, error) {
recordValuePtr := reflect.New(recordType)
- err := json.Unmarshal([]byte(ctx.Event.BodyJSON), recordValuePtr.Interface())
+ err := json.Unmarshal([]byte(ctx.event.BodyJSON), recordValuePtr.Interface())
if err != nil {
return nil, errors.Wrapf(err, "failed to JSON decode message body")
}
diff --git a/queues/handler.go b/queues/handler.go
index aa29674f912e4c9204c356874c429ad2fd4ad3bb..36ba9ec40d196177c7ce54f6e638471ba3935f0a 100644
--- a/queues/handler.go
+++ b/queues/handler.go
@@ -23,8 +23,9 @@ func NewHandler(fnc interface{}) (Handler, error) {
}
//arg[0] must implement interface sqs.IContext
- if _, ok := reflect.New(fncType.In(0)).Interface().(IContext); !ok {
- return h, errors.Errorf("first arg %v does not implement sqs.IContext", fncType.In(0))
+ if fncType.In(0) != contextInterfaceType &&
+ !fncType.In(0).Implements(contextInterfaceType) {
+ return h, errors.Errorf("first arg %v does not implement %v", fncType.In(0), contextInterfaceType)
}
//arg[1] must be a struct for the message record body. It may be an empty struct, but
diff --git a/queues/mem/consumer.go b/queues/mem/consumer.go
index eb314a15de8ee7a79558f02aa231a23a1f923288..dd88a468c0c7ea3318c6d893ae0991354c5b2c72 100644
--- a/queues/mem/consumer.go
+++ b/queues/mem/consumer.go
@@ -1,7 +1,6 @@
package mem
import (
- "context"
"encoding/json"
"fmt"
"math/rand"
@@ -11,39 +10,54 @@ import (
"time"
"github.com/google/uuid"
- "github.com/uptrace/bun"
+ "gitlab.com/uafrica/go-utils/audit"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
"gitlab.com/uafrica/go-utils/queues"
"gitlab.com/uafrica/go-utils/service"
)
-func NewConsumer(routes map[string]interface{}) *Consumer {
+func NewConsumer(routes map[string]interface{}) *consumer {
router, err := queues.NewRouter(routes)
if err != nil {
panic(fmt.Sprintf("cannot create router: %+v", err))
}
- return &Consumer{
- ILogger: logger.New().WithFields(map[string]interface{}{"env": "dev"}).NextColor(),
- router: router,
- queues: map[string]*queue{},
+ //l := logger.New().WithFields(map[string]interface{}{"env": "dev"})
+ //l.IFormatter = l.IFormatter.NextColor()
+ return &consumer{
+ Service: service.New(),
+ //Logger: l,
+ router: router,
+ queues: map[string]*queue{},
}
}
-type Consumer struct {
+type consumer struct {
sync.Mutex
- logger.ILogger
+ service.Service
+ //logger.Logger
router queues.Router
- dbConn service.IDatabaseConnector
queues map[string]*queue
}
-func (consumer *Consumer) WithDb(dbConn service.IDatabaseConnector) queues.IConsumer {
- consumer.dbConn = dbConn
+//wrap Service.WithStarter to return cron, else cannot be chained
+func (consumer *consumer) WithStarter(name string, starter service.IStarter) queues.Consumer {
+ consumer.Service = consumer.Service.WithStarter(name, starter)
return consumer
}
-func (consumer *Consumer) Queue(name string) (*queue, error) {
+//wrap Service.WithErrorReporter to return api, else cannot be chained
+func (consumer *consumer) WithErrorReporter(reporter service.IErrorReporter) queues.Consumer {
+ consumer.Service = consumer.Service.WithErrorReporter(reporter)
+ return consumer
+}
+
+//wrap else cannot be chained
+func (consumer *consumer) WithAuditor(auditor audit.Auditor) queues.Consumer {
+ consumer.Service = consumer.Service.WithAuditor(auditor)
+ return consumer
+}
+
+func (consumer *consumer) Queue(name string) (*queue, error) {
consumer.Lock()
defer consumer.Unlock()
q, ok := consumer.queues[name]
@@ -51,7 +65,7 @@ func (consumer *Consumer) Queue(name string) (*queue, error) {
q = &queue{
consumer: consumer,
name: name,
- ch: make(chan queues.Event),
+ ch: make(chan service.Event),
}
go q.run()
consumer.queues[name] = q
@@ -61,20 +75,20 @@ func (consumer *Consumer) Queue(name string) (*queue, error) {
//do not call this - when using local producer, the consumer is automatically running
//for each queue you send to, and processing from q.run()
-func (consumer *Consumer) Run() {
+func (consumer *consumer) Run() {
panic(errors.Errorf("DO NOT RUN LOCAL CONSUMER"))
}
-func (consumer *Consumer) ProcessFile(filename string) error {
+func (consumer *consumer) ProcessFile(filename string) error {
f, err := os.Open(filename)
if err != nil {
return errors.Wrapf(err, "failed to open queue event file %s", filename)
}
defer f.Close()
- var event queues.Event
+ var event service.Event
if err := json.NewDecoder(f).Decode(&event); err != nil {
- return errors.Wrapf(err, "failed to read queues.Event from file %s", filename)
+ return errors.Wrapf(err, "failed to read service.Event from file %s", filename)
}
q := queue{
@@ -92,9 +106,9 @@ func (consumer *Consumer) ProcessFile(filename string) error {
}
type queue struct {
- consumer *Consumer
+ consumer *consumer
name string
- ch chan queues.Event
+ ch chan service.Event
}
func (q *queue) run() {
@@ -106,40 +120,19 @@ func (q *queue) run() {
}
}
-func (q *queue) process(event queues.Event) error {
+func (q *queue) process(event service.Event) error {
//todo: create context with logger
rand.Seed(time.Now().Unix())
//report handler crashes
- //defer sqs.crashReporter.Catch(ctx)
-
- var db *bun.DB
- if q.consumer.dbConn != nil {
- var err error
- db, err = q.consumer.dbConn.Connect()
- if err != nil {
- return errors.Wrapf(err, "failed to connect to db")
- }
- }
-
- baseCtx := context.Background()
- ctx := queues.Context{
- Context: service.NewContext(baseCtx, map[string]interface{}{
- "env": "dev",
- "request_id": event.RequestID,
- "message_type": event.TypeName,
- }),
- IProducer: q, //todo: q can only send back into this queue... may need to send to other queues!
- Event: event,
- RequestID: event.RequestIDValue,
- DB: db,
+ // if q.crashReporter != nil {
+ // defer q.crashReporter.Catch(ctx)
+ // }
+ ctx, err := queues.NewContext(q.consumer.Service, event)
+ if err != nil {
+ return err
}
- ctx.WithFields(map[string]interface{}{
- "params": event.ParamValues,
- "body": event.BodyJSON,
- }).Infof("Queue(%s) Recv SQS Event: %v", q.name, event)
-
//routing on messageType
sqsHandler, err := q.consumer.router.Route(event.TypeName)
if err != nil {
@@ -161,6 +154,17 @@ func (q *queue) process(event queues.Event) error {
return errors.Wrapf(err, "invalid message body")
}
+ ctx.WithFields(map[string]interface{}{
+ "params": event.ParamValues,
+ "body": event.BodyJSON,
+ }).Infof("RECV(%s) Queue(%s).Type(%s).Due(%s): (%T)%v",
+ "---", //not yet available here - not part of event, and in SQS I think it is passed in SQS layer, so need to extend local channel to include this along with event
+ q.name,
+ event.TypeName,
+ event.DueTime,
+ recordStruct,
+ recordStruct)
+
ctx.Debugf("message (%T) %+v", recordStruct, recordStruct)
args = append(args, reflect.ValueOf(recordStruct))
@@ -168,11 +172,11 @@ func (q *queue) process(event queues.Event) error {
if len(results) > 0 && !results[0].IsNil() {
return errors.Wrapf(results[0].Interface().(error), "handler failed")
}
- ctx.Debugf("handler success")
+ ctx.Debugf("Handler done")
return nil
} //queue.process()
-func (q *queue) Send(event queues.Event) (msgID string, err error) {
+func (q *queue) Send(event service.Event) (msgID string, err error) {
event.MessageID = uuid.New().String()
q.ch <- event
return event.MessageID, nil
diff --git a/queues/mem/producer.go b/queues/mem/producer.go
index 0f90a67df5941ec8078cf495bc9284d9400f4b93..99977c171d7b249b7f6e3afc0d0549824c612afa 100644
--- a/queues/mem/producer.go
+++ b/queues/mem/producer.go
@@ -2,11 +2,11 @@ package mem
import (
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/queues"
+ "gitlab.com/uafrica/go-utils/service"
)
//can only produce locally if also consuming local
-func NewProducer(consumer *Consumer) queues.IProducer {
+func NewProducer(consumer *consumer) service.Producer {
if consumer == nil {
panic(errors.Errorf("cannot product locally without consumer"))
}
@@ -16,10 +16,10 @@ func NewProducer(consumer *Consumer) queues.IProducer {
}
type producer struct {
- consumer *Consumer
+ consumer *consumer
}
-func (producer *producer) Send(event queues.Event) (string, error) {
+func (producer *producer) Send(event service.Event) (string, error) {
q, err := producer.consumer.Queue(event.QueueName)
if err != nil {
return "", errors.Wrapf(err, "failed to get/create queue(%s)", event.QueueName)
diff --git a/queues/producer.go b/queues/producer.go
deleted file mode 100644
index 1f9b09f71df3bfa88c999aec8b44452ef74c7b2a..0000000000000000000000000000000000000000
--- a/queues/producer.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package queues
-
-type IProducer interface {
- Send(event Event) (msgID string, err error)
-}
diff --git a/queues/sqs/consumer.go b/queues/sqs/consumer.go
index 8b9e191f6b2152e6ccf46b9e19bc43daa9c164c5..c14ecde081af1d35cf0cac4d068d8c91cb86077c 100644
--- a/queues/sqs/consumer.go
+++ b/queues/sqs/consumer.go
@@ -15,14 +15,13 @@ import (
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-lambda-go/lambdacontext"
"github.com/google/uuid"
- "github.com/uptrace/bun"
+ "gitlab.com/uafrica/go-utils/audit"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
"gitlab.com/uafrica/go-utils/queues"
"gitlab.com/uafrica/go-utils/service"
)
-func NewConsumer(requestIDHeaderKey string, routes map[string]interface{}) queues.IConsumer {
+func NewConsumer(requestIDHeaderKey string, routes map[string]interface{}) queues.Consumer {
env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
if env == "" {
env = "dev"
@@ -42,27 +41,42 @@ func NewConsumer(requestIDHeaderKey string, routes map[string]interface{}) queue
}
return consumer{
- ILogger: logger.New().WithFields(map[string]interface{}{"env": env}),
+ Service: service.New(),
+ //Logger: logger.New().WithFields(map[string]interface{}{"env": env}),
env: env,
router: router,
requestIDHeaderKey: requestIDHeaderKey,
ConstantMessageType: sqsMessageType,
producer: NewProducer(requestIDHeaderKey),
+ checks: map[string]queues.ICheck{},
}
}
type consumer struct {
- logger.ILogger //for logging outside of context
+ service.Service
env string
router queues.Router
requestIDHeaderKey string
ConstantMessageType string //from os.Getenv("SQS_MESSAGE_TYPE")
- dbConn service.IDatabaseConnector
- producer queues.IProducer
+ producer service.Producer
+ checks map[string]queues.ICheck
}
-func (consumer consumer) WithDb(dbConn service.IDatabaseConnector) queues.IConsumer {
- consumer.dbConn = dbConn
+//wrap Service.WithStarter to return cron, else cannot be chained
+func (consumer consumer) WithStarter(name string, starter service.IStarter) queues.Consumer {
+ consumer.Service = consumer.Service.WithStarter(name, starter)
+ return consumer
+}
+
+//wrap Service.WithErrorReporter to return api, else cannot be chained
+func (consumer consumer) WithErrorReporter(reporter service.IErrorReporter) queues.Consumer {
+ consumer.Service = consumer.Service.WithErrorReporter(reporter)
+ return consumer
+}
+
+//wrap else cannot be chained
+func (consumer consumer) WithAuditor(auditor audit.Auditor) queues.Consumer {
+ consumer.Service = consumer.Service.WithAuditor(auditor)
return consumer
}
@@ -104,16 +118,9 @@ func (consumer consumer) Handler(baseCtx context.Context, lambdaEvent events.SQS
rand.Seed(time.Now().Unix())
//report handler crashes
- //defer sqs.crashReporter.Catch(ctx)
-
- var db *bun.DB
- if consumer.dbConn != nil {
- var err error
- db, err = consumer.dbConn.Connect()
- if err != nil {
- return errors.Wrapf(err, "failed to connect to db")
- }
- }
+ // if consumer.crashReporter != nil {
+ // defer sqs.crashReporter.Catch(ctx)
+ // }
if consumer.ConstantMessageType != "" {
//legacy mode for fixed message type as used in shiplogic
@@ -146,30 +153,25 @@ func (consumer consumer) Handler(baseCtx context.Context, lambdaEvent events.SQS
messageType = *messageTypeAttr.StringValue
}
- ctx := queues.Context{
- Context: service.NewContext(baseCtx, map[string]interface{}{
- "env": consumer.env,
- "request_id": requestID,
- "message_type": messageType,
- }),
- IProducer: consumer.producer, //needed so handler can queue other events or requeue this event
- Event: queues.Event{
- //producer: nil,
- MessageID: message.MessageId,
- QueueName: "N/A", //not sure how to get queue name from lambda Event... would be good to log it, may be in os.Getenv(???)?
- TypeName: messageType,
- DueTime: time.Now(),
- RequestIDValue: requestID,
- BodyJSON: message.Body,
- },
- RequestID: requestID,
- DB: db,
+ event := service.Event{
+ //producer: nil,
+ MessageID: message.MessageId,
+ QueueName: "N/A", //not sure how to get queue name from lambda Event... would be good to log it, may be in os.Getenv(???)?
+ TypeName: messageType,
+ DueTime: time.Now(),
+ RequestIDValue: requestID,
+ BodyJSON: message.Body,
+ }
+
+ ctx, err := queues.NewContext(consumer.Service, event)
+ if err != nil {
+ return err
}
ctx.WithFields(map[string]interface{}{
"message_index": messageIndex,
"message": message,
- }).Infof("Queue(%s) Start SQS Handler Event: %v", ctx.Event.QueueName, ctx.Event)
+ }).Infof("Queue(%s) Start SQS Handler Event: %v", ctx.Event().QueueName, ctx.Event)
//routing on messageType
sqsHandler, err := consumer.router.Route(messageType)
@@ -201,6 +203,7 @@ func (consumer consumer) Handler(baseCtx context.Context, lambdaEvent events.SQS
results := handler.FuncValue.Call(args)
if len(results) > 0 && !results[0].IsNil() {
ctx.Errorf("handler failed: %+v", results[0].Interface().(error))
+ consumer.Service.ReportError(ctx.Data(), err)
}
}
}
diff --git a/queues/sqs/producer.go b/queues/sqs/producer.go
index e5de43248faa42b9e3d353c52c245e1a9631ebd8..8f009fa48788ad4c642af98b4529041019b9c1d9 100644
--- a/queues/sqs/producer.go
+++ b/queues/sqs/producer.go
@@ -11,10 +11,10 @@ import (
"github.com/aws/aws-sdk-go/service/sqs"
"gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
+ "gitlab.com/uafrica/go-utils/service"
)
-func NewProducer(requestIDHeaderKey string) queues.IProducer {
+func NewProducer(requestIDHeaderKey string) service.Producer {
region := os.Getenv("AWS_REGION")
if region == "" {
panic(errors.Errorf("environment AWS_REGION is not defined"))
@@ -26,7 +26,7 @@ func NewProducer(requestIDHeaderKey string) queues.IProducer {
region: region,
requestIDHeaderKey: requestIDHeaderKey,
session: nil,
- queues: map[string]*Messenger{},
+ queues: map[string]*QueueProducer{},
}
}
@@ -35,11 +35,12 @@ type producer struct {
region string
requestIDHeaderKey string
session *session.Session
- queues map[string]*Messenger
+ queues map[string]*QueueProducer
}
// Note: Calling code needs SQS IAM permissions
-func (producer *producer) Send(event queues.Event) (string, error) {
+func (producer *producer) Send(event service.Event) (string, error) {
+ logger.Debugf("producer=%T=%v", producer, producer)
messenger, ok := producer.queues[event.QueueName]
if !ok {
producer.Lock()
@@ -62,7 +63,8 @@ func (producer *producer) Send(event queues.Event) (string, error) {
return "", errors.Wrapf(err, "failed to create AWS session")
}
- messenger = &Messenger{
+ messenger = &QueueProducer{
+ producer: producer,
session: sess,
service: sqs.New(sess),
queueURL: queueURL,
@@ -78,16 +80,16 @@ func (producer *producer) Send(event queues.Event) (string, error) {
}
}
-// Messenger sends an arbitrary message via SQS to a particular queue URL
-type Messenger struct {
+// QueueProducer sends an arbitrary message via SQS to a particular queue URL
+type QueueProducer struct {
producer *producer
session *session.Session
service *sqs.SQS
queueURL string
}
-func (m *Messenger) Send(event queues.Event) (string, error) {
- logger.Debugf("Sending event %+v", event)
+func (m *QueueProducer) Send(event service.Event) (string, error) {
+ //logger.Debugf("Sending event %+v", event)
//add params as message attributes
msgAttrs := make(map[string]*sqs.MessageAttributeValue)
diff --git a/reflection/reflection.go b/reflection/reflection.go
new file mode 100644
index 0000000000000000000000000000000000000000..14faf28a7641715e420fb3a7fa3a396f117cf0ac
--- /dev/null
+++ b/reflection/reflection.go
@@ -0,0 +1,92 @@
+package reflection
+
+import (
+ "reflect"
+ "time"
+
+ "gitlab.com/uafrica/go-utils/logger"
+)
+
+func SetPointerTime(field reflect.Value, value *time.Time) {
+ if !isFieldValid(field, false) {
+ return // Field doesn't exist
+ }
+
+ if field.Kind() != reflect.Ptr {
+ logger.Error("Field need to be *Field")
+ return
+ }
+ field.Set(reflect.ValueOf(value))
+}
+
+func SetInt64(field reflect.Value, value int64) {
+ if !isFieldValid(field, false) {
+ return // Field doesn't exist
+ }
+ if field.Kind() != reflect.Int64 {
+ logger.Error("Claims: Field is not of type Int64")
+ return
+ }
+ if field.OverflowInt(value) {
+ logger.Error("Claims: Int overflow")
+ return
+ }
+ field.SetInt(value)
+}
+
+func SetPointerInt64(field reflect.Value, value *int64) {
+ if !isFieldValid(field, false) {
+ return // Field doesn't exist
+ }
+
+ if field.Kind() != reflect.Ptr {
+ logger.Error("Field need to be *Int64")
+ return
+ }
+
+ field.Set(reflect.ValueOf(value))
+}
+
+func SetString(field reflect.Value, value string) {
+ if !isFieldValid(field, false) {
+ return // Field doesn't exist
+ }
+ if field.Kind() != reflect.String {
+ logger.Error("Claims: Field is not of type String: %v", field.Kind())
+ return
+ }
+ field.SetString(value)
+}
+
+func GetStringValue(field reflect.Value) string {
+ if !isFieldValid(field, true) {
+ return ""
+ }
+
+ if field.Kind() == reflect.String {
+ return field.String()
+ }
+ if field.Kind() == reflect.Ptr && !field.IsNil() {
+ return field.Elem().String()
+ }
+ return ""
+}
+
+func GetInt64Value(field reflect.Value) int64 {
+ if !isFieldValid(field, true) {
+ return 0
+ }
+
+ if field.Kind() == reflect.Int64 {
+ return field.Int()
+ }
+ return 0
+}
+
+func isFieldValid(field reflect.Value, readonly bool) bool {
+ if readonly {
+ return field.IsValid()
+ }
+
+ return field.IsValid() && field.CanSet()
+}
diff --git a/reflection/set.go b/reflection/set.go
new file mode 100644
index 0000000000000000000000000000000000000000..d9a323145a5f324260d076ca4c83007ff42a425d
--- /dev/null
+++ b/reflection/set.go
@@ -0,0 +1,101 @@
+package reflection
+
+import (
+ "fmt"
+ "reflect"
+ "strconv"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+func SetValue(tgt reflect.Value, src interface{}) error {
+ if reflect.TypeOf(src) == tgt.Type() {
+ tgt.Set(reflect.ValueOf(src))
+ return nil
+ }
+
+ //need some kind of type conversion
+ switch tgt.Type().Kind() {
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ //setting a signed integer
+ var i64 int64
+ strValue := fmt.Sprintf("%v", src)
+ if strValue != "" {
+ var err error
+ i64, err = strconv.ParseInt(strValue, 10, 64)
+ if err != nil {
+ return errors.Wrapf(err, "\"%s\" is not a number", strValue)
+ }
+ }
+ switch tgt.Type().Kind() {
+ case reflect.Int:
+ tgt.Set(reflect.ValueOf(int(i64)))
+ case reflect.Int8:
+ tgt.Set(reflect.ValueOf(int8(i64)))
+ case reflect.Int16:
+ tgt.Set(reflect.ValueOf(int16(i64)))
+ case reflect.Int32:
+ tgt.Set(reflect.ValueOf(int32(i64)))
+ case reflect.Int64:
+ tgt.Set(reflect.ValueOf(i64))
+ }
+
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+ //parse to int for this struct field
+ var u64 uint64
+ strValue := fmt.Sprintf("%v", src)
+ if strValue != "" {
+ var err error
+ u64, err = strconv.ParseUint(strValue, 10, 64)
+ if err != nil {
+ return errors.Errorf("\"%s\" is not a number", strValue)
+ }
+ }
+
+ switch tgt.Type().Kind() {
+ case reflect.Uint:
+ tgt.Set(reflect.ValueOf(uint(u64)))
+ case reflect.Uint8:
+ tgt.Set(reflect.ValueOf(uint8(u64)))
+ case reflect.Uint16:
+ tgt.Set(reflect.ValueOf(uint16(u64)))
+ case reflect.Uint32:
+ tgt.Set(reflect.ValueOf(uint32(u64)))
+ case reflect.Uint64:
+ tgt.Set(reflect.ValueOf(u64))
+ }
+
+ case reflect.Bool:
+ strValue := strings.ToLower(fmt.Sprintf("%v", src))
+ if strValue == "true" || strValue == "yes" || strValue == "1" {
+ tgt.Set(reflect.ValueOf(true))
+ } else {
+ tgt.Set(reflect.ValueOf(false))
+ }
+
+ case reflect.String:
+ tgt.Set(reflect.ValueOf(fmt.Sprintf("%v", src)))
+
+ case reflect.Float32:
+ strValue := fmt.Sprintf("%v", src)
+ if f64, err := strconv.ParseFloat(strValue, 32); err != nil {
+ return errors.Wrapf(err, "\"%s\" is not a valid number", strValue)
+ } else {
+ tgt.Set(reflect.ValueOf(float32(f64)))
+ }
+
+ case reflect.Float64:
+ strValue := fmt.Sprintf("%v", src)
+ if f64, err := strconv.ParseFloat(strValue, 64); err != nil {
+ return errors.Wrapf(err, "\"%s\" is not a valid number", strValue)
+ } else {
+ tgt.Set(reflect.ValueOf(f64))
+ }
+
+ default:
+ return errors.Errorf("unsupported type %v", tgt.Type().Kind())
+ } //switch param struct field
+ return nil
+
+}
diff --git a/service/context.go b/service/context.go
index da6d76918b452aab81809346f3ff80639ae76779..d53caf3b3a8b71693d5d81904cbb0d3388d1dcc0 100644
--- a/service/context.go
+++ b/service/context.go
@@ -2,11 +2,56 @@ package service
import (
"context"
+ "reflect"
+ "regexp"
"time"
+ "gitlab.com/uafrica/go-utils/audit"
+ "gitlab.com/uafrica/go-utils/errors"
"gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
+type Context interface {
+ context.Context
+ logger.Logger
+ Producer
+ audit.Auditor
+
+ RequestID() string
+ MillisecondsSinceStart() int64
+ StartTime() time.Time
+
+ //set claim values - things that cannot change - typically UserID, AccountID, Username, ...
+ //the fieldName must be public, i.e. start with uppercase A-Z, no underscores, its not a tag name :-)
+ //once set, it will override any field in params/body struct recursively with this Golang field name (before validation)
+ //Set will fail if the value is already set on this context, i.e. the value cannot change
+ //Call this in your app Start(), before params/body is extracted
+ //value may only be of type int64 or string for now...
+ ClaimSet(fieldName string, value interface{}) error
+
+ //Get can retrieve any claim value
+ ClaimGet(fieldName string) (interface{}, bool)
+
+ //Claim() return all values so you can iterate over them, but not change them...
+ //note: a context.Context also support its own set of values, which you can use as you like
+ //but the idea is that you use this instead, which we apply to params/body
+ Claim() map[string]interface{}
+
+ //context data (names must be snake_case)
+ //unlike claim, any type of value may be stored
+ //but like claims, value can never change
+ WithValue(name string, value interface{}) Context
+ Set(name string, value interface{}) error
+ Get(name string) (interface{}, bool)
+ //Value(name string) interface{} //return nil if not set, inherited from context.Context and overloaded to retrieve local first, just like Get()
+ ValueOrDefault(name string, defaultValue interface{}) interface{}
+ Data() map[string]interface{}
+
+ //write an audit event
+ AuditChange(eventType string, orgValue, newValue interface{})
+}
+
//values: are added to context and logger
//these values are logged for every log event in this context
//values can be added later using with value, but won't be logged
@@ -14,48 +59,172 @@ import (
// authentication may set the user_id etc... and other package may retrieve it but not change it
type valueKey string
-func NewContext(base context.Context, values map[string]interface{}) Context {
+func (s service) NewContext(base context.Context, requestID string, values map[string]interface{}) (Context, error) {
+ if values == nil {
+ values = map[string]interface{}{}
+ }
+ values["request-id"] = requestID
+
for n, v := range values {
base = context.WithValue(base, valueKey(n), v)
}
- return Context{
+ l := logger.New().WithFields(values)
+ l.IFormatter = l.IFormatter.NextColor()
+
+ ctx := &serviceContext{
Context: base,
- Logger: logger.New().WithFields(values).NextColor(),
+ Logger: l,
+ Producer: s.Producer,
+ Auditor: s.Auditor,
startTime: time.Now(),
+ requestID: requestID,
+ data: map[string]interface{}{},
+ claim: map[string]interface{}{},
}
-}
-// type IContext interface {
-// context.Context
-// logger.ILogger
-// StartTime() time.Time
-// MillisecondsSinceStart() int64
-// ValueOrDefault(name string, defaultValue interface{}) interface{}
-// WithValue(name string, value interface{}) Context
-// }
+ for starterName, starter := range s.starters {
+ var starterData interface{}
+ starterData, err := starter.Start(ctx)
+ if err != nil {
+ ctx.Errorf("Start(%s) failed: %+v ...", starterName, err)
+ return nil, errors.Wrapf(err, "%s", starterName)
+ }
+ if err = ctx.Set(starterName, starterData); err != nil {
+ ctx.Errorf("Start(%s) failed to set (%T)%+v: %+v ...", starterName, starterData, starterData, err)
+ return nil, errors.Wrapf(err, "failed to set starter(%s) data=(%T)%+v", starterName, starterData, starterData)
+ }
+ ctx.Debugf("Start(%s)=(%T)%+v", starterName, starterData, starterData)
+ }
+ return ctx, nil
+}
-type Context struct {
+type serviceContext struct {
context.Context
logger.Logger
+ Producer
+ audit.Auditor
startTime time.Time
+ requestID string
+ claim map[string]interface{}
+ data map[string]interface{}
+}
+
+func (ctx serviceContext) RequestID() string {
+ return ctx.requestID
+}
+
+const claimFieldNamePattern = `[A-Z][a-zA-Z0-9]*`
+
+var claimFieldNameRegex = regexp.MustCompile("^" + claimFieldNamePattern + "$")
+
+func (ctx *serviceContext) ClaimSet(fieldName string, value interface{}) error {
+ if !claimFieldNameRegex.MatchString(fieldName) {
+ return errors.Errorf("invalid claim field name \"%s\"", fieldName)
+ }
+ if oldValue, exists := ctx.claim[fieldName]; exists {
+ return errors.Errorf("ClaimSet(%s=(%T)%v) failed because already set to (%T)%v", fieldName, value, value, oldValue, oldValue)
+ }
+ switch reflect.TypeOf(value).Kind() {
+ case reflect.Int64:
+ case reflect.String:
+ default:
+ panic(errors.Errorf("claim(%s)=(%T)%v is neither sting nor int64", fieldName, value, value))
+ }
+ ctx.claim[fieldName] = value
+ return nil
+}
+
+func (ctx serviceContext) ClaimGet(fieldName string) (interface{}, bool) {
+ if cv, ok := ctx.claim[fieldName]; ok {
+ return cv, true
+ }
+ return nil, false
+}
+
+func (ctx serviceContext) Claim() map[string]interface{} {
+ claim := map[string]interface{}{}
+ for n, v := range ctx.claim {
+ claim[n] = v
+ }
+ return claim
+}
+
+func (ctx *serviceContext) WithValue(name string, value interface{}) Context {
+ ctx.Context = context.WithValue(ctx.Context, valueKey(name), value)
+ return ctx
}
-func (ctx Context) MillisecondsSinceStart() int64 {
+func (ctx *serviceContext) Set(name string, value interface{}) error {
+ if !string_utils.IsSnakeCase(name) {
+ return errors.Errorf("invalid context value name \"%s\"", name)
+ }
+ if oldValue, exists := ctx.data[name]; exists {
+ return errors.Errorf("Set(%s=(%T)%v) failed because already set to (%T)%v", name, value, value, oldValue, oldValue)
+ }
+ ctx.data[name] = value
+ return nil
+}
+
+func (ctx *serviceContext) Get(name string) (interface{}, bool) {
+ if cv, ok := ctx.data[name]; ok {
+ return cv, true
+ }
+ //alternative: try value from context.Context
+ if cv := ctx.Context.Value(name); cv != nil {
+ return cv, true
+ }
+ return nil, false
+}
+
+//Value override context.Context.Value to retrieve first from our own data
+func (ctx *serviceContext) Value(key interface{}) interface{} {
+ if name, ok := key.(string); ok {
+ if cv, ok := ctx.data[name]; ok {
+ return cv
+ }
+ }
+ //alternative: try value from context.Context
+ if cv := ctx.Context.Value(key); cv != nil {
+ return cv
+ }
+ return nil
+}
+
+func (ctx *serviceContext) Data() map[string]interface{} {
+ data := map[string]interface{}{}
+ for n, v := range ctx.data {
+ data[n] = v
+ }
+ return data
+}
+
+func (ctx *serviceContext) MillisecondsSinceStart() int64 {
return time.Since(ctx.startTime).Milliseconds()
}
-func (ctx Context) StartTime() time.Time {
+func (ctx *serviceContext) StartTime() time.Time {
return ctx.startTime
}
-func (ctx Context) ValueOrDefault(name string, defaultValue interface{}) interface{} {
+func (ctx *serviceContext) ValueOrDefault(name string, defaultValue interface{}) interface{} {
if value := ctx.Value(valueKey(name)); value != nil {
return value
}
return defaultValue
}
-func (ctx Context) WithValue(name string, value interface{}) Context {
- ctx.Context = context.WithValue(ctx.Context, valueKey(name), value)
- return ctx
+func (ctx *serviceContext) AuditChange(eventType string, orgValue, newValue interface{}) {
+ event, err := audit.NewEvent(
+ ctx.Claim()["username"].(string), //use username as source (will default to "SYSTEM" if undefined)
+ eventType,
+ orgValue,
+ newValue,
+ )
+ if err != nil {
+ ctx.Errorf("failed to define audit event: %+v", err)
+ return
+ }
+ if err := ctx.Auditor.WriteEvent(ctx.requestID, event); err != nil {
+ ctx.Errorf("failed to audit change: %+v", err)
+ }
}
diff --git a/service/database.go b/service/database.go
deleted file mode 100644
index 3670ee57c9dab156075ab1bd3ce6ffca1a0e27d4..0000000000000000000000000000000000000000
--- a/service/database.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package service
-
-import "github.com/uptrace/bun"
-
-type IDatabaseConnector interface {
- Connect() (*bun.DB, error)
-}
diff --git a/queues/event.go b/service/event.go
similarity index 86%
rename from queues/event.go
rename to service/event.go
index 6f4e0f960798c8004ead9eb06b77ddb421001675..e68e84d02d9d934e538a9e88e4a06b682f1bce3f 100644
--- a/queues/event.go
+++ b/service/event.go
@@ -1,4 +1,4 @@
-package queues
+package service
import (
"encoding/json"
@@ -9,12 +9,12 @@ import (
"gitlab.com/uafrica/go-utils/logger"
)
-type IProducerLogger interface {
- IProducer
- logger.ILogger
+type ProducerLogger interface {
+ Producer
+ logger.Logger
}
-func NewEvent(producer IProducerLogger, queueName string) Event {
+func NewEvent(producer ProducerLogger, queueName string) Event {
if producer == nil {
panic(errors.Errorf("NewEvent(producer=nil)"))
}
@@ -31,7 +31,7 @@ func NewEvent(producer IProducerLogger, queueName string) Event {
}
type Event struct {
- producer IProducerLogger
+ producer ProducerLogger
MessageID string //assigned by implementation (AWS/mem/..)
QueueName string //queue determine sequencing, items in same queue are delivered one-after-the-other, other queues may deliver concurrent to this queue
TypeName string //type determines which handler processes the event
@@ -82,9 +82,9 @@ func (event Event) Params(params map[string]string) Event {
}
func (event Event) Send(value interface{}) (string, error) {
- event.producer.Debugf("Queue(%s) Send SQS Event: %v",
- event.QueueName,
- event)
+ if event.producer == nil {
+ return "", errors.Errorf("Send with producer==nil")
+ }
if value != nil {
jsonBody, err := json.Marshal(value)
if err != nil {
@@ -92,12 +92,20 @@ func (event Event) Send(value interface{}) (string, error) {
}
event.BodyJSON = string(jsonBody)
}
- if event.producer == nil {
- return "", errors.Errorf("Send with producer==nil")
- }
+ event.producer.Debugf("Queue(%s) Sending SQS Event: %v",
+ event.QueueName,
+ event)
+
if msgID, err := event.producer.Send(event); err != nil {
return "", errors.Wrapf(err, "failed to send event")
} else {
+ event.producer.Infof("SENT(%s) Queue(%s).Type(%s).Due(%s): (%T)%v",
+ msgID,
+ event.QueueName,
+ event.TypeName,
+ event.DueTime,
+ value,
+ value)
return msgID, nil
}
}
diff --git a/service/producer.go b/service/producer.go
new file mode 100644
index 0000000000000000000000000000000000000000..050c56f8134cd77432b3383e827d163933f60aa8
--- /dev/null
+++ b/service/producer.go
@@ -0,0 +1,7 @@
+package service
+
+//Producer sends an event for async processing
+type Producer interface {
+ Send(event Event) (msgID string, err error)
+ //todo: method to request an event after some delay with incrementing attempt nr
+}
diff --git a/service/service.go b/service/service.go
new file mode 100644
index 0000000000000000000000000000000000000000..92e45a245ab2b9dac1b34223eb5bda9117b1683b
--- /dev/null
+++ b/service/service.go
@@ -0,0 +1,106 @@
+package service
+
+import (
+ "context"
+ "os"
+
+ "gitlab.com/uafrica/go-utils/audit"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/string_utils"
+)
+
+type Service interface {
+ logger.Logger
+ IErrorReporter
+ audit.Auditor
+ WithStarter(name string, starter IStarter) Service
+ WithProducer(producer Producer) Service
+ WithAuditor(auditor audit.Auditor) Service
+ WithErrorReporter(reporter IErrorReporter) Service
+ NewContext(base context.Context, requestID string, values map[string]interface{}) (Context, error)
+}
+
+func New() Service {
+ env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
+ if env == "" {
+ env = "dev"
+ }
+ return service{
+ Logger: logger.New().WithFields(map[string]interface{}{"env": env}),
+ IErrorReporter: DoNotReportErrors{},
+ Auditor: audit.None(),
+ env: env,
+ starters: map[string]IStarter{},
+ }
+}
+
+type service struct {
+ logger.Logger //for logging outside of context
+ Producer //for sending async events
+ IErrorReporter
+ audit.Auditor
+ env string
+ starters map[string]IStarter
+}
+
+func (s service) Env() string {
+ return s.env
+}
+
+//adds a starter function to call in each new context
+//they will be called in the sequence they were added (before api/cron/queue checks)
+//and they do not have details about the event
+//if starter returns error, processing fails
+//if starter succeeds, and return !=nil data, it is stored against the name
+// so your handler can retieve it with:
+// checkData := ctx.Value(name).(expectedType)
+// or
+// checkData,ok := ctx.Value(name).(expectedType)
+// if !ok { ... }
+//you can implement one starter that does everything and return a struct or
+//implement one for your db, one for rate limit, one for ...
+//the name must be snake-case, e.g. "this_is_my_starter_name"
+func (s service) WithStarter(name string, starter IStarter) Service {
+ if !string_utils.IsSnakeCase(name) {
+ panic(errors.Errorf("invalid starter name=\"%s\", expecting snake_case names only", name))
+ }
+ if starter == nil {
+ panic(errors.Errorf("starter(%s)==nil", name))
+ }
+ if _, ok := s.starters[name]; ok {
+ panic(errors.Errorf("starter(%s) already defined", name))
+ }
+ s.starters[name] = starter
+ return s
+}
+
+func (s service) WithProducer(producer Producer) Service {
+ if producer != nil {
+ s.Producer = producer
+ }
+ return s
+}
+
+func (s service) WithErrorReporter(reporter IErrorReporter) Service {
+ if reporter == nil {
+ panic(errors.Errorf("ErrorReporter==nil"))
+ }
+ s.IErrorReporter = reporter
+ return s
+}
+
+func (s service) WithAuditor(auditor audit.Auditor) Service {
+ if auditor != nil {
+ s.Auditor = auditor
+ }
+ return s
+}
+
+type IErrorReporter interface {
+ ReportError(fields map[string]interface{}, err error)
+}
+
+type DoNotReportErrors struct{}
+
+func (DoNotReportErrors) ReportError(fields map[string]interface{}, err error) {}
diff --git a/service/start.go b/service/start.go
new file mode 100644
index 0000000000000000000000000000000000000000..c30c11733164a66a642b6d89f1ce1f264525e465
--- /dev/null
+++ b/service/start.go
@@ -0,0 +1,8 @@
+package service
+
+type IStarter interface {
+ //called at the start of api/cron/queues processing, before checks, e.g. to ensure we have db connection
+ //i.e. setup things that does not depend on the request/event details
+ //if you need the request details, you need to implement a check for each of the api, cron and/or queue as needed, not a Start() method.
+ Start(ctx Context) (interface{}, error)
+}
diff --git a/string_utils/snake.go b/string_utils/snake.go
new file mode 100644
index 0000000000000000000000000000000000000000..358d7fa5e2e503e1f417fc507bd659cbbb30f8f2
--- /dev/null
+++ b/string_utils/snake.go
@@ -0,0 +1,11 @@
+package string_utils
+
+import "regexp"
+
+const snakeCasePattern = `[a-z]([a-z0-9_]*[a-z0-9])*`
+
+var snakeCaseRegex = regexp.MustCompile("^" + snakeCasePattern + "$")
+
+func IsSnakeCase(name string) bool {
+ return snakeCaseRegex.MatchString(name)
+}