From d14e15cc870310f1fcd461bbd2f7eb2eaa934cbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?France=CC=81=20Wilke?= <francewilke@gmail.com>
Date: Fri, 10 Dec 2021 13:06:55 +0200
Subject: [PATCH] SL refactor
---
.DS_Store | Bin 0 -> 10244 bytes
api/README.md | 215 -------------
api/api.go | 147 ---------
api/check.go | 5 -
api/context.go | 153 ---------
api/cors.go | 5 -
api/crash.go | 5 -
api/handler.go | 83 -----
api/lambda.go | 296 ------------------
api/params_test.go | 202 ------------
api/router.go | 90 ------
api/test.go | 1 -
api_documentation/api_documentation.go | 166 ++++++++++
{logs => api_logs}/api-logs.go | 52 ++-
api_responses/api_responses.go | 256 +++++++++++++++
audit/{change.go => audit.go} | 110 ++-----
cognito/cognito.go | 135 ++++++++
cognito/words.go | 104 ++++++
compare/compare.go | 17 +
compare/compare_test.go | 56 ++++
consumer/README.md | 4 -
consumer/check.go | 7 -
consumer/consumer.go | 10 -
consumer/context.go | 69 ----
consumer/handler.go | 52 ---
consumer/mem_consumer/README.md | 3 -
consumer/mem_consumer/consumer.go | 186 -----------
consumer/mem_consumer/producer.go | 44 ---
consumer/router.go | 76 -----
consumer/sqs_consumer/consumer.go | 210 -------------
cron/check.go | 5 -
cron/context.go | 35 ---
cron/cron.go | 122 --------
cron/handler.go | 52 ---
cron/lambda.go | 72 -----
cron/router.go | 65 ----
date_utils/date_utils.go | 118 +++++++
encryption/encryption.go | 22 ++
examples/core/api/main.go | 79 -----
examples/core/app/app.go | 33 --
examples/core/app/users/users.go | 156 ---------
examples/core/cron/main.go | 66 ----
examples/core/db/database.go | 38 ---
examples/core/email/notify.go | 17 -
examples/core/sqs/main.go | 13 -
go.mod | 43 +--
go.sum | 66 ++--
handler_utils/api.go | 178 +++++++++++
handler_utils/cron.go | 23 ++
handler_utils/handler.go | 114 +++++++
handler_utils/request.go | 32 ++
handler_utils/sqs.go | 60 ++++
logger/context.go | 44 ---
logger/db.go | 24 --
logger/format.go | 151 ---------
logger/global.go | 86 -----
logger/level.go | 36 ---
logger/logger.go | 238 --------------
logger/logs_test.go | 57 ----
logs/logs.go | 243 ++++++++++++++
logs/logs_test.go | 32 ++
logs/sensitive_words.go | 71 +++++
.../sensitive_words_test.go | 8 +-
{logger => logs}/stack.go | 4 +-
number_utils/number_utils.go | 26 ++
queues/README.md | 162 ----------
queues/event.go | 112 -------
queues/producer.go | 7 -
queues/sqs_producer/README.md | 3 -
queues/sqs_producer/producer.go | 133 --------
redis/redis.go | 232 ++++++--------
redis/redis_test.go | 145 ---------
reflection/reflection.go | 21 +-
responses/responses.go | 121 +++++++
s3/s3.go | 266 ++++++++++++++++
search/README.md | 24 +-
search/document_store.go | 13 +-
search/document_store_test.go | 8 +-
search/search_test.go | 26 +-
search/time_series.go | 125 ++------
search/writer.go | 4 +-
secrets_manager/secrets_manager.go | 124 ++++++++
service/README.md | 182 -----------
service/context.go | 236 --------------
service/service.go | 77 -----
service/start.go | 8 -
sqs/sqs.go | 112 +++++++
string_utils/string_utils.go | 39 ++-
struct_utils/named_values_to_struct.go | 5 +-
struct_utils/named_values_to_struct_test.go | 14 +-
utils/utils.go | 166 ++++++++++
91 files changed, 2760 insertions(+), 4793 deletions(-)
create mode 100644 .DS_Store
delete mode 100644 api/README.md
delete mode 100644 api/api.go
delete mode 100644 api/check.go
delete mode 100644 api/context.go
delete mode 100644 api/cors.go
delete mode 100644 api/crash.go
delete mode 100644 api/handler.go
delete mode 100644 api/lambda.go
delete mode 100644 api/params_test.go
delete mode 100644 api/router.go
delete mode 100644 api/test.go
create mode 100644 api_documentation/api_documentation.go
rename {logs => api_logs}/api-logs.go (90%)
create mode 100644 api_responses/api_responses.go
rename audit/{change.go => audit.go} (54%)
create mode 100644 cognito/cognito.go
create mode 100644 cognito/words.go
create mode 100644 compare/compare.go
create mode 100644 compare/compare_test.go
delete mode 100644 consumer/README.md
delete mode 100644 consumer/check.go
delete mode 100644 consumer/consumer.go
delete mode 100644 consumer/context.go
delete mode 100644 consumer/handler.go
delete mode 100644 consumer/mem_consumer/README.md
delete mode 100644 consumer/mem_consumer/consumer.go
delete mode 100644 consumer/mem_consumer/producer.go
delete mode 100644 consumer/router.go
delete mode 100644 consumer/sqs_consumer/consumer.go
delete mode 100644 cron/check.go
delete mode 100644 cron/context.go
delete mode 100644 cron/cron.go
delete mode 100644 cron/handler.go
delete mode 100644 cron/lambda.go
delete mode 100644 cron/router.go
create mode 100644 date_utils/date_utils.go
create mode 100644 encryption/encryption.go
delete mode 100644 examples/core/api/main.go
delete mode 100644 examples/core/app/app.go
delete mode 100644 examples/core/app/users/users.go
delete mode 100644 examples/core/cron/main.go
delete mode 100644 examples/core/db/database.go
delete mode 100644 examples/core/email/notify.go
delete mode 100644 examples/core/sqs/main.go
create mode 100644 handler_utils/api.go
create mode 100644 handler_utils/cron.go
create mode 100644 handler_utils/handler.go
create mode 100644 handler_utils/request.go
create mode 100644 handler_utils/sqs.go
delete mode 100644 logger/context.go
delete mode 100644 logger/db.go
delete mode 100644 logger/format.go
delete mode 100644 logger/global.go
delete mode 100644 logger/level.go
delete mode 100644 logger/logger.go
delete mode 100644 logger/logs_test.go
create mode 100644 logs/logs.go
create mode 100644 logs/logs_test.go
create mode 100644 logs/sensitive_words.go
rename logger/logger_test.go => logs/sensitive_words_test.go (93%)
rename {logger => logs}/stack.go (98%)
create mode 100644 number_utils/number_utils.go
delete mode 100644 queues/README.md
delete mode 100644 queues/event.go
delete mode 100644 queues/producer.go
delete mode 100644 queues/sqs_producer/README.md
delete mode 100644 queues/sqs_producer/producer.go
delete mode 100644 redis/redis_test.go
create mode 100644 responses/responses.go
create mode 100644 s3/s3.go
create mode 100644 secrets_manager/secrets_manager.go
delete mode 100644 service/README.md
delete mode 100644 service/context.go
delete mode 100644 service/service.go
delete mode 100644 service/start.go
create mode 100644 sqs/sqs.go
create mode 100644 utils/utils.go
diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..634d4d473f7934c869c9ebc383a9e0bd2d463931
GIT binary patch
literal 10244
zcmZQzU|@7AO)+F(P+(wS;9!8z0^AG?3~dYy5+IrZ0YLH~j1Ymcm_g<+Ffe2=FfcGP
zFbFU(<T0c$<S?W%6fwj@)!?I;p!!)D7#MgNG8ht3^p`S}Fk~|1FcjzJr?@2L<R^ig
z`r@eF!L1t)JAxgJ&+JjT(GVC7fzc2c4S~@R7!3jJA;1U`=0L0OA)=$?Xb6mkz=#e3
zQ27ArAb@BGXj{DjLW6`D7#SEqO?VIuYQlrViV@roU;v4Mw1Q}mRuB!+%D@O>fz1GG
zWng50Xk~=9=ovxX2(Ua@I|CyFSUUqF16Vr)BiLpJ21bZ>21bZ>21aNPg;4<P3<gGs
zb_Pa>c8K68IT`|^A%GGBruh5b1sR6H$@#ej5IqSXiiIJEA)g_gp*SZUS<c}t0|NtG
zE;ru=-e+IM<CDK(?O{h$c|@Ncn>?z$VC#7p@)$}Pav74K;|uWd0aSM|Zew6zz^NA5
zUBLoyci~Zwa3?Y9QQZwLi&z<o7*ZKh7&6gAjPEuB0|SyQvfFrt;ci1wfp86W6<F-&
z0^6Sk9+yc5kE!G{<e`R{=_UpS2CV9k-Q+3;cM}e^2-gu(i|Wn*P`cs-yR(>~fFYkD
zkD-_$6+OK8;}{qiu&ByKN^7B#a5rLAiEtr)m8k9m$0HL%F@rH`xXC_a09Ap-493WA
z=#zoF0jvjk{0dzT)gEx2#Rm3!B0~{FGD8NceP=-qL6Jwc_xRk8U%+Ey3=C*$5WdEx
z2E`qW2B7%h2fHH~94jRZ#SHNbx!@7HM22)w4n%dCKq3PJ0|6b#ZoOm)4<mxQ5UwX)
z7m9ztqnr%f48;sN44Dkc45<w93^+;;rmGALxKtv$KUq_;RuxjF;V>6D)(NXdbuTEI
z8JHQ084A(!0QY_HSSwft**&%P@bHA{L$)7N7K?p648;s33`Go?40#OcxI;)H8C<^N
zP>bv)*OJGVCD((?eVk?^yOJ37sO|>0?O<k?GJr}qoG}~v6v^EfYLVT2{g2vjXg<Pe
zHnO{kQIG0wa8s5QR~i&x2iFVm6pHM&rcii1Bk4zW4R#f%_JhkAP#$MsFaq19$B@a8
z!jQ_4h?=XVU!%$*yX5ty;4RQlVr78qM|KHz6{s!&*X-;JdJJU@#SF;|`5;%Ix`7GQ
z&_`2&><%dnd00-wP=zQtaH~Rf4>;%XG9)sjFr<J>zG8662agL>xAEJ7TTeJuBfImE
z1>Bu@%tsDC;&fnfzaT>*Ljgl3Lp(zYLq0<?xYd-(kjGHMkchqh7dXhkz(9m9Wd9WG
z@rJePh|-CO3(|C=`VX9PIpF@o+7eaR3YsZGSA*>Kr;Fh=C5BnZF@R4Us=L57CMaGZ
z;SOr?;c3Z8)<M$^4%NtRopcT!&Unm6b~SN2P~8s>S5}5ZhEi}#wFEVt^G1Nn0k|x(
z`xae-yAMe}vU{+rK(!y7@7Td5KsrMnLncEBLq4h-gg}i%G!@A1xOW5Y4h&U@w1-<2
zs(ZlU0Wl|+p@0F@8cRiW56e#m1_m@0$nH_Rhvpu*ImlsyTNM`fa51ESb1<Iv3uidE
zC67fNvfEl0z|%igGm%|LNG+;6!STq&04i-jEyqfR0_-IY<2tZAv8Y3KXW<ig7-BUO
z*`0*cVsR%Ms2$5t#E{QWgx=2T1(k>>^2qKwEvf*kbI{ZvQW`EbsO~rbs)g~)eWRv1
z<4|z9foG;1(d&lKl!NEcQEdgMZcyJ7WIKZbX#C%mfdSY2KUmQyF&YA+Auv2cfRV)|
z*u@Dv=#Jfkpe8t|%m9u5fyUp#r4~dzhzlCh2Ppy3;Nf^C$f6<*kZFtz4B*BQlnw5(
xF)~0#^9O7)05lc}sk`B=4J;)jL<PLwMlFj+>wn1VEYxrtt^Z*w)dqY04*)-ecGv&_
literal 0
HcmV?d00001
diff --git a/api/README.md b/api/README.md
deleted file mode 100644
index 1435a49..0000000
--- a/api/README.md
+++ /dev/null
@@ -1,215 +0,0 @@
-# API
-## Overview
-API is a ```syncronous``` [service](../service/README.md), meaning the user makes an HTTP(S) request and waits for the response. Handlers are associated with an HTTP method and path and should return ASAP. If there are any long running processes associated with your handler, they should be implemented as asynchronous services using [queues](../queues/README.md).
-
-## Contents
-* [Usage](#usage)
-* [Examples](#examples)
-
-## Usage
-The main function of your API must:
-
-1) Create the API and specify:
- - the key used for request-id in HTTP headers,
- - the HTTP routes (paths, methods and handlers)
-
- Example:
-
- api.New("uafrica-request-id", apiRoutes())
-
- The routes are defined as a map of path with a map of method storing the handler, e.g.:
-
- func apiRoutes() map[string]map[string]interface{} {
- return map[string]map[string]interface{}{
- "/users":{
- "GET": users.GETHandler,
- "POST": users.POSTHandler,
- },
- "/shipments":{
- "PATCH": shipments.PATCHHandler,
- },
- }
- }
-
- The handler function signature is discussed [here](#handlers).
-
-2) Define Starters (optional)
-
- See [Service Starters](../service/README.MD#define-starters)
-
-3) Define Checks (optional)
-
- See [Service Checks](../service/README.MD#define-checks)
-
-4) Define Events Handlers (for testing only)
-
- If the API makes async calls, i.e. if any of your handlers are sending queued events to SQS, then you could also define the event handlers in your API for local testing. When you do and you run locally, the SQS producer/consumer will be replaced by a in-memory implementation using Go channels. This means that your API will not send SQS event, but queue the processing internally, allowing you to do full end-to-end testing and even stepping through the async event handler code as part of your API debugger.
-
- If you do not define the events, the API will still be able to send queued events, but they will always go to AWS SQS.
-
- To define events, do:
-
- .WithEvents(myEvents)
-
- Where for example:
-
- myEvents = map[string]interface{} {
- "add-user": users.AddHandler,
- "del-user": users.DelHandler,
- }
-
-6) Run()
-
- Now the service definition is complete, it must run.
-
- This will run as AWS Lambda unless the local port is defined, then it will start to listen on the specified TCP port for HTTP traffic.
-
-## Example
-See [example1/api/api.go](../examples/core/api/main.go)
-
-# Handlers
-Each handler function referenced in your routes must have the following signature:
-
-```
-func MyHandler(
- ctx api.Context,
- params MyParams,
- body MyBody,
-) (
- res MyRes,
- err error,
-) {
- ...
-}
-```
-
-Notes:
-* ```params``` is required but may be empty struct{} for handlers that does not expect any URL parameters.
-* ```body``` may be omitted for handlers that does not process an HTTP body.
-* ```res``` may be omitted for handlers that does not respond with content.
-
-The API do support legacy handlers also with the following signature. In this case, the handler has no context or other benefits from the framework. These handers should be phased out.
-
- func (
- ctx lambda.Context,
- req events.APIGatewayProxyRequest,
- ) (
- events.APIGatewayProxyResponse,
- error,
- ) {
- ...
- }
-
-See the following regarding struct types used in handlers:
-* [Struct Types](../service/README.md#struct-types)
-* [Validate()](../service/README.md#validate)
-
-
-## URL Parameter Structs
-
-On top of above rules regarding input struct, the following applies specifically to API parameter structs.
-
-Body struct are populated with ```json.Unmarshal()``` but params has a manual parsing from the URL that does not implement all types. In generally we use only strings and int64 types for fields in the params struct. This may change in future releases.
-
-The json tag of a field must be used in the URL as param name.
-
-Arrays of strings or int64 may also be used for parameters, e.g.:
-
- type MyParams struct {
- IDs []int64 `json:"ids"`
- }
-
-Then a URL may specify either:
-* ```?ids=[1,2,3]```, or
-* ```?ids=1&ids=2&ids=3```
-
-Embedded structs may also be used, e.g. a generic struct for paging with Offset and Limit, embedded into a get struct that adds an ID, and embed that into a handler specified type with more parameters, as in this example:
-
- type GenericPageParams struct {
- Limit int64 `json:"limit"`
- Offset int64 `json:"offset"`
- }
-
- type GenericGetParams struct {
- ID int64 `json:"id"`
- GenericPageParams
- }
-
- type MyGetParams struct {
- GenericGetParams
- Name string `json:"name"`
- }
-
-Call the embedded validation in the parent struct:
-
- func (p MyGetParams) Validate() error {
- if p.Name == "" { return errors.Errorf("missing name") }
- if err := p.GenericGetParams.Validate(); err != nil { return err }
- return nil
- }
-
- func (p GenericGetParams) Validate() error {
- ...
- if err := p.GenericPageParams.Validate(); err != nil { return err }
- return nil
- }
-
-In future, we could use a tag to automate embedded validation ...
-
-
-## QueryStringParameters()
-
-Some legacy functions operator from parameters in a map, like ```QueryStringParameters()``` returning ```map[string]string```.
-
-That can still be used but is discorouged because:
-* AWS API Gateway puts multiple params in another map - which you will miss when reading only that.
-* Params structs parses both and supports array as multiple params or as CSV
-* Params structs restrics the code from using adhoc params that are not documented anywhere
-* Params structs enables us to generate documentation
-* Params structs parses strings to int, meaning your code does not have to.
-
-In future, we will extend the struct parsing to support many more features, including validation of values, out of the box, not requiring code in Validate() method. The pain for you is only to list all your params in a struct, then you will keep on getting benefits... avoid using maps!
-
-If you still call functions that expects a map, you can convert your struct to a map using ```struct_utils.MapParams(myParamsStructValue)```.
-
-
-## Errors
-When a handler cannot deal with an error, it returns an error. The response value will be discarded, so the handler can just return an empty struct of the correct type.
-
-The error could be any go error, which will result in an HTTP 500 Internal Server Error.
-
-To return another HTTP error code, use:
-
- return MyRes{}, errors.HTTP(code, err, message)
-
-Note: The ```err != nil``` is required but the message may be "". If a message is added, it will wrap the error in another layer.
-
-If the error is passed up through a few layers, and multiple HTTP codes are defined, the lowest code in the error stack will be returned to the user.
-
-
-
-# Local Testing
-
-When API is executed in local environment, with:
-* env LOG_LEVEL=debug, and
-* api.WithEvents(app.QueueRoutes())
-Then any async events are sent into golang channels and processed locally, without delay. This allows you do debug and set break points in async services inside your api executable.
-
-When executing the api from the console using a local port, the Lambda function is not used and instead the API runs in a normal http server on the specified port. Since it is not running with mage, it will load config.local.json to complete the environment normally defined in mage/cdk.
-
-# API Documentation
-
-API documentation will eventually be generated from doc tags in your structs. This is not yet complete...
-
-
-# API Logger
-
-API Logs are written from global variables using logs.LogIncomingAPIRequest()
-
-# Router Path Parameters
-
-It is not yet possible to use path parameters, e.g. ```/user/{user_id}```.
-
-# Automated Testing
-
-Automated Testing is not yet part of the code base. It should be...
diff --git a/api/api.go b/api/api.go
deleted file mode 100644
index 718cc6f..0000000
--- a/api/api.go
+++ /dev/null
@@ -1,147 +0,0 @@
-package api
-
-import (
- "fmt"
- "os"
- "runtime/debug"
-
- "github.com/aws/aws-lambda-go/lambda"
- "gitlab.com/uafrica/go-utils/audit"
- "gitlab.com/uafrica/go-utils/consumer/mem_consumer"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logs"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/queues/sqs_producer"
- "gitlab.com/uafrica/go-utils/service"
- "gitlab.com/uafrica/go-utils/string_utils"
-)
-
-// Ctx extends service ctx to include url etc.
-var Ctx Context
-
-// New creates the API with the specified routes keys on [path][method]
-// value could be any of the handler function signatures supported by the api.Router
-// requestIDHeaderKey is defined in the response header to match the requestID from the request
-func New(requestIDHeaderKey string, routes map[string]map[string]interface{}) Api {
- if requestIDHeaderKey == "" {
- requestIDHeaderKey = "request-id"
- }
-
- router, err := NewRouter(routes)
- if err != nil {
- panic(fmt.Sprintf("cannot create router: %+v", err))
- }
-
- return Api{
- Service: service.New(),
- router: router,
- requestIDHeaderKey: requestIDHeaderKey,
- checks: map[string]ICheck{},
- crashReporter: defaultCrashReporter{},
- cors: nil,
- localQueueEventHandlers: nil,
- }
-}
-
-type Api struct {
- service.Service
- router Router
- requestIDHeaderKey string
- checks map[string]ICheck
- crashReporter ICrashReporter
- cors ICORS
- localQueueEventHandlers map[string]interface{} //only applies when running locally for local in-memory queues
-}
-
-//wrap Service.WithStarter to return api, else cannot be chained
-func (api Api) WithStarter(name string, starter service.Starter) Api {
- api.Service = api.Service.WithStarter(name, starter)
- return api
-}
-
-//wrap else cannot be chained
-func (api Api) WithProducer(producer queues.Producer) Api {
- api.Service = api.Service.WithProducer(producer)
- return api
-}
-
-//add a check to startup of each context
-//they will be called in the sequence they were added
-//if check return error, processing stops and err is returned
-//if check succeed, and return !=nil data, it is stored against the name
-// so your handler can retieve it with:
-// checkData := ctx.Value(name).(expectedType)
-// or
-// checkData,ok := ctx.Value(name).(expectedType)
-// if !ok { ... }
-//you can implement one check that does everything and return a struct or
-//implement one for your db, one for rate limit, one for auth, one for ...
-//the name must be snake-case, e.g. "this_is_my_check_data_name"
-func (api Api) WithCheck(name string, check ICheck) Api {
- if !string_utils.IsSnakeCase(name) {
- panic(errors.Errorf("invalid check name=\"%s\", expecting snake_case names only", name))
- }
- if check == nil {
- panic(errors.Errorf("check(%s) func==nil", name))
- }
- if _, ok := api.checks[name]; ok {
- panic(errors.Errorf("check(%s) already defined", name))
- }
- api.checks[name] = check
- return api
-}
-
-func (api Api) WithCORS(cors ICORS) Api {
- if cors != nil {
- api.cors = cors
- }
- return api
-}
-
-func (api Api) WithCrashReported(crashReporter ICrashReporter) Api {
- if crashReporter != nil {
- api.crashReporter = crashReporter
- }
- return api
-}
-
-//WithEvents are not used in production, only when env LOG_LEVEL=debug
-//then the SQS producer is replaced with in-memory producer that uses
-//go channels to queue and process events, so they can be debugged locally
-func (api Api) WithEvents(eventHandlers map[string]interface{}) Api {
- if api.localQueueEventHandlers != nil {
- panic("local queue event handlers already defined")
- }
- api.localQueueEventHandlers = eventHandlers
- return api
-}
-
-//run and panic on error
-func (api Api) Run() {
- //decide local or SQS
- var producer queues.Producer
- if (os.Getenv("LOG_LEVEL") == "debug") && api.localQueueEventHandlers != nil {
- //use in-memory channels for async events
- api.Debugf("Using in-memory channels for async events ...")
- producer = mem_consumer.NewProducer(mem_consumer.New(api.Service, api.localQueueEventHandlers))
- } else {
- //use SQS for async events
- api.Debugf("Using SQS queue producer for async events ...")
- producer = sqs_producer.New(api.requestIDHeaderKey)
- }
- api = api.WithProducer(producer)
- audit.Init(producer)
- logs.Init(producer)
-
- //run as an AWS Lambda function
- lambda.Start(api.Handler)
-}
-
-type defaultCrashReporter struct{}
-
-func (defaultCrashReporter) Catch(ctx Context) {
- crashErr := recover()
- if crashErr != nil {
- ctx.Errorf("crashed: %v, with stack: %s", crashErr, string(debug.Stack()))
- }
-}
diff --git a/api/check.go b/api/check.go
deleted file mode 100644
index b7be7f6..0000000
--- a/api/check.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package api
-
-type ICheck interface {
- Check(Context) (interface{}, error)
-}
diff --git a/api/context.go b/api/context.go
deleted file mode 100644
index 31253eb..0000000
--- a/api/context.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package api
-
-import (
- "encoding/json"
- "reflect"
-
- "github.com/aws/aws-lambda-go/events"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/reflection"
- "gitlab.com/uafrica/go-utils/service"
- "gitlab.com/uafrica/go-utils/struct_utils"
-)
-
-type Context interface {
- service.Context
- Request() events.APIGatewayProxyRequest
- GetRequestParams(paramsStructType reflect.Type) (interface{}, error)
- GetRequestBody(requestStructType reflect.Type) (interface{}, error)
- LogAPIRequestAndResponse(res events.APIGatewayProxyResponse, err error)
-}
-
-type apiContext struct {
- service.Context
- request events.APIGatewayProxyRequest
-}
-
-func (ctx apiContext) Request() events.APIGatewayProxyRequest {
- return ctx.request
-}
-
-//todo: change to be a ctx method that defer to log so it does not have to be called explicitly
-//it should also capture metrics for the handler and automaticlaly write the audit record,
-//(but still allow for audit to be suppressed may be in some cases)
-func (ctx *apiContext) LogAPIRequestAndResponse(res events.APIGatewayProxyResponse, err error) {
- fields := map[string]interface{}{
- "path": ctx.request.Path,
- "method": ctx.request.HTTPMethod,
- "status_code": res.StatusCode,
- "api_gateway_request_id": ctx.RequestID(),
- }
-
- if ctx.request.HTTPMethod == "GET" {
- fields["req-query"] = ctx.request.QueryStringParameters
- }
-
- statusOK := res.StatusCode >= 200 && res.StatusCode <= 299
- if err != nil || !statusOK {
- fields["error"] = err
- fields["req-body"] = ctx.request.Body
- fields["req-query"] = ctx.request.QueryStringParameters
- fields["res-body"] = res.Body
- }
- ctx.Context.WithFields(fields).Infof("Request & Response: err=%+v", err)
-}
-
-//allocate struct for params, populate it from the URL parameters then validate and return the struct
-func (ctx apiContext) GetRequestParams(paramsStructType reflect.Type) (interface{}, error) {
- paramsStructValuePtr := reflect.New(paramsStructType)
- nv := struct_utils.NamedValuesFromURL(ctx.request.QueryStringParameters, ctx.request.MultiValueQueryStringParameters)
- unused, err := struct_utils.UnmarshalNamedValues(nv, paramsStructValuePtr.Interface())
- if err != nil {
- return nil, errors.Wrapf(err, "invalid parameters")
- }
- if len(unused) > 0 {
- logger.Warnf("Unknown parameters: %+v", unused)
- }
- if err := ctx.applyClaim("params", paramsStructValuePtr.Interface()); err != nil {
- return nil, errors.Wrapf(err, "failed to fill claims on params")
- }
- if validator, ok := paramsStructValuePtr.Interface().(IValidator); ok {
- if err := validator.Validate(); err != nil {
- return nil, errors.Wrapf(err, "invalid params")
- }
- }
- return paramsStructValuePtr.Elem().Interface(), nil
-}
-
-func (ctx apiContext) GetRequestBody(requestStructType reflect.Type) (interface{}, error) {
- requestStructValuePtr := reflect.New(requestStructType)
- err := json.Unmarshal([]byte(ctx.request.Body), requestStructValuePtr.Interface())
- if err != nil {
- return nil, errors.Wrapf(err, "failed to JSON request body")
- }
-
- if err := ctx.applyClaim("body", requestStructValuePtr.Interface()); err != nil {
- return nil, errors.Wrapf(err, "failed to fill claims on body")
- }
- if validator, ok := requestStructValuePtr.Interface().(IValidator); ok {
- if err := validator.Validate(); err != nil {
- return nil, errors.Wrapf(err, "invalid request body")
- }
- }
- return requestStructValuePtr.Elem().Interface(), nil
-}
-
-type IValidator interface {
- Validate() error
-}
-
-func (ctx *apiContext) applyClaim(name string, valuePtr interface{}) error {
- t := reflect.TypeOf(valuePtr)
- if t.Kind() != reflect.Ptr {
- return errors.Errorf("%T is not a pointer", valuePtr) //programming error... it must be a pointer to be able to change it
- }
- t = t.Elem()
- if t.Kind() != reflect.Struct {
- //ctx.Debugf("Not setting claims on %T", valuePtr)
- return nil //not a struct - nothing to do - is allowed e.g. for posting a string.
- }
- if err := ctx.setClaim(name, t, reflect.ValueOf(valuePtr).Elem()); err != nil {
- return errors.Wrapf(err, "failed to set claim on %T", valuePtr)
- }
- return nil
-}
-
-func (ctx *apiContext) setClaim(name string, structType reflect.Type, structValue reflect.Value) error {
- if len(ctx.Claim()) == 0 {
- ctx.Debugf("NO CLAIM to apply to %s of type (%s)", name, structType.Name())
- return nil
- }
-
- for fieldName, claimValue := range ctx.Claim() {
- if field := structValue.FieldByName(fieldName); field.IsValid() {
- if err := reflection.SetValue(field, claimValue); err != nil {
- return errors.Errorf("failed to set %s.%s=(%T)%v", structType.Name(), fieldName, claimValue, claimValue)
- }
- ctx.Debugf("defined claim %s.%s=(%T)%v ...", name, fieldName, claimValue, claimValue)
- // } else {
- // ctx.Debugf("claim(%s) does not apply to %s", fieldName, structType.Name())
- }
- }
-
- //recurse into sub-structs and sub struct ptrs (not yet slices)
- for i := 0; i < structType.NumField(); i++ {
- f := structType.Field(i)
- if len(f.Name) > 0 && f.Name[0] >= 'a' && f.Name[0] <= 'z' {
- //private field - do not enter
- continue
- }
- if f.Type.Kind() == reflect.Struct {
- if err := ctx.setClaim(name+"."+structType.Field(i).Name, f.Type, structValue.Field(i)); err != nil {
- return errors.Wrapf(err, "failed to set claim on sub struct %s.%s", structType.Name(), f.Name)
- }
- }
- if f.Type.Kind() == reflect.Ptr && f.Type.Elem().Kind() == reflect.Struct && !structValue.Field(i).IsNil() {
- if err := ctx.setClaim(name+"."+structType.Field(i).Name, f.Type.Elem(), structValue.Field(i).Elem()); err != nil {
- return errors.Wrapf(err, "failed to set claim on sub &struct %s.%s", structType.Name(), f.Name)
- }
- }
- }
- return nil
-}
diff --git a/api/cors.go b/api/cors.go
deleted file mode 100644
index 4d36770..0000000
--- a/api/cors.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package api
-
-type ICORS interface {
- CORS() map[string]string //return CORS headers
-}
diff --git a/api/crash.go b/api/crash.go
deleted file mode 100644
index bf686ea..0000000
--- a/api/crash.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package api
-
-type ICrashReporter interface {
- Catch(ctx Context) //Report(method string, path string, crash interface{})
-}
diff --git a/api/handler.go b/api/handler.go
deleted file mode 100644
index 5829cb2..0000000
--- a/api/handler.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package api
-
-import (
- "reflect"
-
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type handler struct {
- RequestParamsType reflect.Type
- RequestBodyType reflect.Type
- ResponseType reflect.Type
- FuncValue reflect.Value
-}
-
-func NewHandler(fnc interface{}) (handler, error) {
- h := handler{}
-
- fncType := reflect.TypeOf(fnc)
- if fncType.NumIn() < 1 || fncType.NumIn() > 2 {
- return h, errors.Errorf("takes %d args instead of (Params[, Body])", fncType.NumIn())
- }
- if fncType.NumOut() < 1 || fncType.NumOut() > 2 {
- return h, errors.Errorf("returns %d results instead of ([Response,] error)", fncType.NumOut())
- }
-
- //arg[0] must be a struct for params. It may be an empty struct, but
- //all public fields require a json tag which we will use to math the URL param name
- if err := validateStructType(fncType.In(0)); err != nil {
- return h, errors.Wrapf(err, "first arg %v is not valid params struct type", fncType.In(0))
- }
- h.RequestParamsType = fncType.In(0)
-
- //arg[1] is optional and must be a struct for request body. It may be an empty struct, but
- //all public fields require a json tag which we will use to unmarshal the request body from JSON
- if fncType.NumIn() >= 2 {
- if fncType.In(1).Kind() == reflect.Slice {
- if err := validateStructType(fncType.In(1).Elem()); err != nil {
- return h, errors.Errorf("second arg %v is not valid body []struct type", fncType.In(1))
- }
- } else {
- if err := validateStructType(fncType.In(1)); err != nil {
- return h, errors.Errorf("second arg %v is not valid body struct type", fncType.In(1))
- }
- }
-
- //todo: check special fields for claims, and see if also applies to params struct...
- //AccountID must be int64 or *int64 with tag =???
- //UserID must be int64 or *int64 with tag =???
- //Username must be string with tag =???
-
- h.RequestBodyType = fncType.In(1)
- }
-
- //last result must be error
- if _, ok := reflect.New(fncType.Out(fncType.NumOut() - 1)).Interface().(*error); !ok {
- return h, errors.Errorf("last result %v is not error type", fncType.Out(fncType.NumOut()-1))
- }
-
- h.FuncValue = reflect.ValueOf(fnc)
- return h, nil
-}
-
-func validateStructType(t reflect.Type) error {
- if t.Kind() != reflect.Struct {
- return errors.Errorf("%v is %v, not a struct", t, t.Kind())
- }
- for i := 0; i < t.NumField(); i++ {
- f := t.Field(i)
- if f.Name[0] >= 'a' && f.Name[0] <= 'z' {
- //lowercase fields should not have json tag
- if f.Tag.Get("json") != "" {
- return errors.Errorf("%s.%s must be uppercase because it has a json tag \"%s\"",
- t.Name(),
- f.Name,
- f.Tag.Get("json"))
- }
- }
-
- // if f.... check tags recursively... for now, not too strict ... add checks if we see issues that break the API, to help dev to fix before we deploy, or to prevent bad habits...
- }
- return nil
-}
diff --git a/api/lambda.go b/api/lambda.go
deleted file mode 100644
index 19edeae..0000000
--- a/api/lambda.go
+++ /dev/null
@@ -1,296 +0,0 @@
-package api
-
-import (
- "context"
- "database/sql"
- "encoding/json"
- "fmt"
- "math/rand"
- "net/http"
- "reflect"
- "time"
-
- "github.com/aws/aws-lambda-go/events"
- "github.com/aws/aws-lambda-go/lambdacontext"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/logs"
-)
-
-func (api Api) NewContext(baseCtx context.Context, requestID string, request events.APIGatewayProxyRequest) (Context, error) {
- serviceContext, err := api.Service.NewContext(baseCtx, requestID, nil)
- if err != nil {
- return nil, err
- }
-
- return &apiContext{
- Context: serviceContext,
- request: request,
- }, nil
-}
-
-//this is native handler for lambda passed into lambda.Start()
-//to run locally, this is called from app.ServeHTTP()
-func (api Api) Handler(baseCtx context.Context, apiGatewayProxyReq events.APIGatewayProxyRequest) (res events.APIGatewayProxyResponse, err error) {
- res = events.APIGatewayProxyResponse{
- StatusCode: http.StatusInternalServerError,
- Body: "undefined response",
- Headers: map[string]string{},
- }
-
- // Replace the proxy resource with the path, has some edge cases but works for our current API implementation
- // Edge case being that if have path params then specify those routes explicitly
- if apiGatewayProxyReq.Resource == "/{proxy+}" {
- apiGatewayProxyReq.Resource = apiGatewayProxyReq.Path
- }
-
- //get request-id from HTTP headers (used when making internal service calls)
- //if not defined in header, get the AWS request id from the AWS context
- requestID, ok := apiGatewayProxyReq.Headers[api.requestIDHeaderKey]
- if !ok || requestID == "" {
- if lambdaContext, ok := lambdacontext.FromContext(baseCtx); ok && lambdaContext != nil {
- requestID = lambdaContext.AwsRequestID
- }
- }
-
- // service context invoke the starters and could fail, e.g. if cannot connect to db
- Ctx, err = api.NewContext(baseCtx, requestID, apiGatewayProxyReq)
- if err != nil {
- return res, err
- }
-
- //report handler crashes
- if api.crashReporter != nil {
- defer api.crashReporter.Catch(Ctx)
- }
-
- defer func() {
- //set CORS headers on every response
- if api.cors != nil {
- for n, v := range api.cors.CORS() {
- res.Headers[n] = v
- }
- }
- }()
-
- defer func() {
- Ctx.LogAPIRequestAndResponse(res, err)
- if err != nil {
- Ctx.Errorf("failed: %+v", err)
-
- //try to retrieve HTTP code from error
- if withCause, ok := err.(errors.ErrorWithCause); ok && withCause.Code() != 0 {
- res.StatusCode = withCause.Code()
- err = withCause.Cause() //drop the http layers + up
- } else {
- //no HTTP code indicate in err,
- //see if there are SQL errors that could indicate code
- if code, ok := StatusCodeFromSQLError(err); ok {
- res.StatusCode = code
- }
- }
-
- errorMessage := fmt.Sprintf("%c", err)
- jsonError, _ := json.Marshal(map[string]interface{}{"message": errorMessage})
- res.Headers["Content-Type"] = "application/json"
- res.Body = string(jsonError)
- err = nil //never pass error back to lambda or http server
- }
- if api.requestIDHeaderKey != "" {
- res.Headers[api.requestIDHeaderKey] = Ctx.RequestID()
- }
- if err := logs.LogIncomingAPIRequest(Ctx.StartTime(), Ctx.RequestID(), Ctx.Claim(), apiGatewayProxyReq, res); err != nil {
- Ctx.Errorf("failed to log: %+v", err)
- }
- }()
-
- //Early return OPTIONS call
- if apiGatewayProxyReq.HTTPMethod == "OPTIONS" {
- res.StatusCode = http.StatusNoContent
- err = nil
- return
- }
-
- rand.Seed(time.Now().Unix())
-
- for checkName, check := range api.checks {
- var checkData interface{}
- checkData, err = check.Check(Ctx)
- if err != nil {
- err = errors.Wrapf(err, "%s", checkName)
- return
- }
- if err = Ctx.Set(checkName, checkData); err != nil {
- err = errors.Wrapf(err, "failed to set check(%s) data=(%T)%+v", checkName, checkData, checkData)
- return
- }
- }
-
- Ctx.Tracef("HTTP %s %s ...\n", apiGatewayProxyReq.HTTPMethod, apiGatewayProxyReq.Resource)
- Ctx.WithFields(map[string]interface{}{
- "http_method": Ctx.Request().HTTPMethod,
- "path": Ctx.Request().Path,
- "api_gateway_request_id": Ctx.Request().RequestContext.RequestID,
- "user_cognito_auth_provider": Ctx.Request().RequestContext.Identity.CognitoAuthenticationProvider,
- "user_arn": Ctx.Request().RequestContext.Identity.UserArn,
- }).Infof("Start API Handler")
-
- //TODO:
- // // Get claims and check the status of the user
- // Ctx.Claims, err = api.RetrieveClaims(&apiGatewayProxyReq)
- // if err != nil {
- // return events.APIGatewayProxyResponse{
- // StatusCode: http.StatusBadRequest,
- // Body: fmt.Sprintf("%v\n", err),
- // Headers: utils.CorsHeaders(),
- // }, nil
- // }
-
- // if Ctx.Claims.UserID != nil {
- // userStatusResponse := checkUserStatus(Ctx.Claims)
- // if userStatusResponse != nil {
- // return *userStatusResponse, nil
- // }
- // }
-
- // permissionString := fmt.Sprintf("API_%s%s:%s", os.Getenv("MICRO_SERVICE_API_BASE_PATH"), apiGatewayProxyReq.Resource, apiGatewayProxyReq.HTTPMethod)
- // if !permissions.HasPermission(Ctx.Claims.Role, permissionString) {
- // response, _ := apierr.ClientError(http.StatusUnauthorized, fmt.Sprintf("You do not have access to the requested resource: %s", permissionString))
- // if Ctx.Claims.Role == nil {
- // Ctx.Errorf("%d :: %s: %v", *Ctx.Claims.RoleID, permissionString, fmt.Errorf("you have no role"))
- // } else if Ctx.Claims.RoleID == nil {
- // Ctx.Errorf("%s: you have no role ID", permissionString)
- // }
- // return response, nil
- // }
-
- //route on method and path
- resourceHandler, err := api.router.Route(apiGatewayProxyReq.Resource, apiGatewayProxyReq.HTTPMethod)
- if err != nil {
- err = errors.Wrapf(err, "invalid route")
- return
- }
-
- if legacyHandlerFunc, ok := resourceHandler.(func(req events.APIGatewayProxyRequest) (response events.APIGatewayProxyResponse, err error)); ok {
- Ctx.Debugf("Calling legacy handler...")
- return legacyHandlerFunc(apiGatewayProxyReq)
- }
-
- handler, ok := resourceHandler.(handler)
- if !ok {
- //should not get here if validateAPIEndpoints() is properly checking!
- err = errors.HTTP(http.StatusInternalServerError, errors.Errorf("invalid handler function %T", resourceHandler), "invalid routing")
- }
-
- //new type of handler function
- //allocate, populate and validate params struct
- paramsStruct, paramsErr := Ctx.GetRequestParams(handler.RequestParamsType)
- if paramsErr != nil {
- err = errors.HTTP(http.StatusBadRequest, paramsErr, "invalid parameters")
- return
- }
- //apply claims to params struct - TODO: Removed - see if cannot force to get claims from context always
- // if err = Ctx.Claims.FillOnObject(Ctx.request, ¶msStruct); err != nil {
- // err = errors.HTTP(http.StatusInternalServerError, err, "claims failed on parameters")
- // return
- // }
- Ctx.Debugf("Params: (%T) %+v", paramsStruct, paramsStruct)
-
- args := []reflect.Value{
- reflect.ValueOf(paramsStruct),
- }
-
- var bodyStruct interface{}
- if handler.RequestBodyType != nil {
- //allocate, populate and validate request struct
- bodyStruct, err = Ctx.GetRequestBody(handler.RequestBodyType)
- if err != nil {
- err = errors.HTTP(http.StatusBadRequest, err, "invalid body")
- return
- }
-
- //apply claims to request struct - TODO: Removed - see if cannot force to get claims from context always
- // if err = Ctx.Claims.FillOnObject(Ctx.request, &bodyStruct); err != nil {
- // err = errors.HTTP(http.StatusInternalServerError, err, "claims failed on body")
- // return
- // }
-
- Ctx.Tracef("Body: (%T) %+v", bodyStruct, bodyStruct)
- args = append(args, reflect.ValueOf(bodyStruct))
- }
-
- //call handler in a func with defer to catch potential crash
- Ctx.Infof("Calling handle %s %s ...", apiGatewayProxyReq.HTTPMethod, apiGatewayProxyReq.Resource)
- var results []reflect.Value
- results, err = func() (results []reflect.Value, err error) {
- defer func() {
- if crashErr := recover(); crashErr != nil {
- stack := logger.CallStack()
- err = errors.Errorf("handler function crashed: %v, with stack: %+v", crashErr, stack)
- return
- }
- }()
- results = handler.FuncValue.Call(args)
- return results, nil
- }()
- if err != nil {
- err = errors.Wrapf(err, "handler failed")
- return
- }
-
- //Ctx.Debugf("handler -> results: %v", results)
- //see if handler failed using last result of type error
- lastResultValue := results[len(results)-1]
- if !lastResultValue.IsNil() {
- err = lastResultValue.Interface().(error)
- if err != nil {
- return
- }
- }
-
- //handler succeeded, some handler does not have a response data (typically post/put/patch/delete)
- err = nil
- switch apiGatewayProxyReq.HTTPMethod {
- case http.MethodDelete:
- res.StatusCode = http.StatusNoContent
- default:
- res.StatusCode = http.StatusOK
- }
-
- if len(results) > 1 {
- responseStruct := results[0].Interface()
- Ctx.Debugf("Response type: %T", responseStruct)
- if responseString, ok := responseStruct.(string); ok {
- res.Headers["Content-Type"] = "application/json"
- res.Body = responseString
- } else {
- var bodyBytes []byte
- bodyBytes, err = json.Marshal(responseStruct)
- if err != nil {
- err = errors.Wrapf(err, "failed to encode response content")
- return
- }
- res.Headers["Content-Type"] = "application/json"
- res.Body = string(bodyBytes)
- }
- } else {
- //no content
- delete(res.Headers, "Content-Type")
- res.Body = ""
- }
- return
-}
-
-//look for SQL errors in the error stack
-func StatusCodeFromSQLError(err error) (int, bool) {
- if err == sql.ErrNoRows {
- return http.StatusNotFound, true
- }
-
- if errWithCause, ok := err.(errors.ErrorWithCause); ok {
- return StatusCodeFromSQLError(errWithCause.Cause())
- }
-
- //could not determine known SQL error
- return 0, false
-}
diff --git a/api/params_test.go b/api/params_test.go
deleted file mode 100644
index b9586e9..0000000
--- a/api/params_test.go
+++ /dev/null
@@ -1,202 +0,0 @@
-package api_test
-
-import (
- "context"
- "encoding/json"
- "reflect"
- "testing"
- "time"
-
- "github.com/aws/aws-lambda-go/events"
- "gitlab.com/uafrica/go-utils/api"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-type P1 struct {
- A int `json:"a"`
-}
-
-type P2 struct {
- P1 //nested struct must be filled
- B int `json:"b"`
- F []int `json:"f"`
-}
-
-type P3 struct {
- P2 //nessted struct must be filled
- C int `json:"c"`
- E []int `json:"e"`
-}
-
-func TestNested(t *testing.T) {
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.SetGlobalFormat(logger.NewConsole())
- var ctx api.Context
- var err error
- ctx, err = api.New("request-id", nil).NewContext(
- context.Background(),
- "123",
- //all URL params are specified as string values
- events.APIGatewayProxyRequest{
- QueryStringParameters: map[string]string{
- "a": "1", //must be written into P3.P2.P1.A
- "b": "2", //must be written into P3.P2.B
- "c": "3", //must be written into P3.C
- "d": "4", //ignored because no field tagged "d"
- },
- MultiValueQueryStringParameters: map[string][]string{
- "e": {"5", "6", "7"}, //filled into P3.E as []string {"5", "6", "7"}
- "f": {"6", "7", "8"}, //filled into P2 as []string {"6", "7", "8"}
- },
- })
- if err != nil {
- t.Fatalf("ERROR: %+v", err)
- }
-
- if p3d, err := ctx.GetRequestParams(reflect.TypeOf(P3{})); err != nil {
- t.Fatalf("ERROR: %+v", err)
- } else {
- p3 := p3d.(P3)
- t.Logf("p3: %+v", p3)
- if p3.C != 3 || p3.B != 2 || p3.A != 1 {
- t.Fatalf("wrong values")
- }
- if len(p3.E) != 3 || p3.E[0] != 5 || p3.E[1] != 6 || p3.E[2] != 7 {
- t.Fatalf("wrong values")
- }
- if len(p3.F) != 3 || p3.F[0] != 6 || p3.F[1] != 7 || p3.F[2] != 8 {
- t.Fatalf("wrong values")
- }
- }
-}
-
-type ParamTypes struct {
- GetParams
- Nr int64 `json:"nr"`
- Name string `json:"name"`
- NrOpt *int64 `json:"nr_opt"`
- NameOpt *string `json:"name_opt"`
- Time1 time.Time `json:"time1"`
- Time2 *time.Time `json:"time2"`
- Dur1 time.Duration `json:"dur1"`
- Dur2 *time.Duration `json:"dur2"`
-
- //lists of values
- NrList []int64 `json:"nrs"`
- NameList []string `json:"names"`
- NrOptList []*int64 `json:"nrs_opt"`
- NameOptList []*string `json:"names_opt"`
- Time1List []time.Time `json:"time1s"`
- Time2List []*time.Time `json:"time2s"`
- Dur1List []time.Duration `json:"dur1s"`
- Dur2List []*time.Duration `json:"dur2s"`
-}
-
-func TestTypes(t *testing.T) {
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.SetGlobalFormat(logger.NewConsole())
- var ctx api.Context
- var err error
- ctx, err = api.New("request-id", nil).NewContext(
- context.Background(),
- "123",
- //all URL params are specified as string values
- events.APIGatewayProxyRequest{
- QueryStringParameters: map[string]string{
- "nr": "1",
- "name": "name2",
- "nr_opt": "3",
- "name_opt": "name4",
- "limit": "5",
- "time1": "2021-11-23T00:00:00+00:00",
- "time2": "2021-11-23T00:00:00+00:00",
- "dur1": "4", //nanoseconds
- "dur2": "4", //nanoseconds
- "nrs": "[1,2,3]",
- "nrs_opt": "[4,5,6]",
- "names": "[A,B,C]",
- "names_opt": "[D,E,F]",
- "time1s": "[2021-11-23T00:00:00+00:00]",
- "dur1s": "[4,5,6]", //nanoseconds
- },
- MultiValueQueryStringParameters: map[string][]string{
- "dur2s": {"11", "12", "13"},
- "time2s": {"2021-11-23T00:00:00+00:00", "2021-11-23T00:00:00+00:00", "2021-11-23T00:00:00+00:00"},
- },
- })
- if err != nil {
- t.Fatalf("ERROR: %+v", err)
- }
-
- if pd, err := ctx.GetRequestParams(reflect.TypeOf(ParamTypes{})); err != nil {
- t.Fatalf("ERROR: %+v", err)
- } else {
- p := pd.(ParamTypes)
- t.Logf("p: %+v", p)
- if p.Nr != 1 || p.Name != "name2" || p.NrOpt == nil || *p.NrOpt != 3 || p.NameOpt == nil || *p.NameOpt != "name4" || p.Limit != 5 {
- t.Errorf("Wrong values: %+v", p)
- }
- jsonParams, _ := json.Marshal(p)
- t.Logf("params: %s", string(jsonParams))
- }
-}
-
-type PageParams struct {
- Limit int64 `json:"limit"`
- Offset int64 `json:"offset"`
-}
-
-type GetParams struct {
- PageParams
- ID int64 `json:"id"`
-}
-
-type MyGetParams struct {
- GetParams
- Search string `json:"search"`
- Find string `json:"find"`
- Find1 []string `json:"find1"`
- Find2 []string `json:"find2"`
- Find3 []string `json:"find3"`
-}
-
-func TestGet(t *testing.T) {
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.SetGlobalFormat(logger.NewConsole())
- var ctx api.Context
- var err error
- ctx, err = api.New("request-id", nil).NewContext(
- context.Background(),
- "123",
- //all URL params are specified as string values
- events.APIGatewayProxyRequest{
- QueryStringParameters: map[string]string{
- "id": "1",
- "limit": "2",
- "offset": "3",
- "search": "4", //single value parts into string
- "find1": "sarel", //single value parsed into array
- "find3": "[hendrik,,\"frederik\",\"johan,johan\"]", //array of 4 values in CSV notation
- },
- MultiValueQueryStringParameters: map[string][]string{
- "find2": {"hans", "gert"}, //multi-values parsed into array
- "find": {"koos"}, //field of type string can be parsed from one multi-value
- },
- })
- if err != nil {
- t.Fatalf("ERROR: %+v", err)
- }
-
- if p3d, err := ctx.GetRequestParams(reflect.TypeOf(MyGetParams{})); err != nil {
- t.Fatalf("ERROR: %+v", err)
- } else {
- get := p3d.(MyGetParams)
- t.Logf("get: %+v", get)
- if get.ID != 1 || get.Offset != 3 || get.Limit != 2 || get.Search != "4" || get.Find != "koos" ||
- len(get.Find1) != 1 || get.Find1[0] != "sarel" ||
- len(get.Find2) != 2 || get.Find2[0] != "hans" || get.Find2[1] != "gert" ||
- len(get.Find3) != 4 || get.Find3[0] != "hendrik" || get.Find3[1] != "" || get.Find3[2] != "frederik" || get.Find3[3] != "johan,johan" {
- t.Fatalf("wrong values")
- }
- }
-}
diff --git a/api/router.go b/api/router.go
deleted file mode 100644
index 5095e9b..0000000
--- a/api/router.go
+++ /dev/null
@@ -1,90 +0,0 @@
-package api
-
-import (
- "fmt"
- "net/http"
-
- "github.com/aws/aws-lambda-go/events"
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type Router struct {
- endpoints map[string]map[string]interface{}
-}
-
-func (r Router) Endpoints() map[string]map[string]interface{} {
- return r.endpoints
-}
-
-func (r Router) Route(path, method string) (interface{}, error) {
- if methods, ok := r.endpoints[path]; !ok {
- return nil, errors.HTTP(http.StatusNotFound, errors.Errorf("%s not found", path), "unknown resource path")
- } else {
- if handler, ok := methods[method]; !ok {
- return nil, errors.HTTP(http.StatusMethodNotAllowed, errors.Errorf("%s not allowed on %s", method, path), "method not allowed")
- } else {
- return handler, nil
- }
- }
-}
-
-//check that all API endpoints are correctly defined using one of the supported handler types
-//return updated endpoints with additional information
-func NewRouter(endpoints map[string]map[string]interface{}) (Router, error) {
- countLegacy := 0
- countHandler := 0
- for resource, methodHandlers := range endpoints {
- if resource == "" {
- return Router{}, errors.Errorf("blank resource")
- }
- if resource == "/api-docs" {
- return Router{}, errors.Errorf("%s may not be a defined endpoint - it is reserved", resource)
- }
- for method, handlerFunc := range methodHandlers {
- switch method {
- case "GET":
- case "POST":
- case "PUT":
- case "PATCH":
- case "DELETE":
- default:
- return Router{}, errors.Errorf("nvalid method:\"%s\" on resource \"%s\"", method, resource)
- }
- if handlerFunc == nil {
- return Router{}, errors.Errorf("nil handler on %s %s", method, resource)
- }
-
- if _, ok := handlerFunc.(func(req events.APIGatewayProxyRequest) (response events.APIGatewayProxyResponse, err error)); ok {
- //ok - leave as is - we support this legacyHandler
- fmt.Printf("%10s %s: OK (legacy handler)\n", method, resource)
- countLegacy++
- } else {
- handler, err := NewHandler(handlerFunc)
- if err != nil {
- return Router{}, errors.Wrapf(err, "%s %s has invalid handler %T", method, resource, handlerFunc)
- }
- //replace the endpoint value so we can quickly call this handler
- endpoints[resource][method] = handler
- fmt.Printf("%10s %s: OK (params: %v, request: %v)\n", method, resource, handler.RequestParamsType, handler.RequestBodyType)
- countHandler++
- }
- }
- }
- fmt.Printf("Checked %d legacy and %d new handlers\n", countLegacy, countHandler)
-
- //add reserved endpoint to generate documentation
- r := Router{
- endpoints: endpoints,
- }
-
- // {
- // docsHandler, err := NewHandler(GETApiDocs(r)) //endpoints))
- // if err != nil {
- // return Router{}, errors.Wrapf(err, "failed to define handler for docs")
- // }
- // endpoints["/api-docs"] = map[string]interface{}{
- // "GET": docsHandler,
- // }
- // }
- return r, nil
-}
diff --git a/api/test.go b/api/test.go
deleted file mode 100644
index 778f64e..0000000
--- a/api/test.go
+++ /dev/null
@@ -1 +0,0 @@
-package api
diff --git a/api_documentation/api_documentation.go b/api_documentation/api_documentation.go
new file mode 100644
index 0000000..ba8a9b5
--- /dev/null
+++ b/api_documentation/api_documentation.go
@@ -0,0 +1,166 @@
+package api_documentation
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/handler_utils"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+type NoParams struct{}
+
+type Docs struct {
+ Paths map[string]DocPath `json:"paths"`
+}
+
+type DocPath struct {
+ Methods map[string]DocMethod `json:"methods"`
+}
+
+type DocMethod struct {
+ Description string `json:"description"`
+ Parameters map[string]DocParam `json:"parameters,omitempty"`
+ Request interface{} `json:"request,omitempty"`
+ Response interface{} `json:"response,omitempty"`
+}
+
+type DocParam struct {
+ Name string
+ Type string
+ Description string
+}
+
+func GetDocs(endpointHandlers map[string]map[string]interface{}) (Docs, error) {
+ docs := Docs{
+ Paths: map[string]DocPath{},
+ }
+ for path, methods := range endpointHandlers {
+ docPath := DocPath{
+ Methods: map[string]DocMethod{},
+ }
+ for method, methodHandler := range methods {
+ docMethod := DocMethod{}
+ if handler, ok := methodHandler.(handler_utils.Handler); !ok {
+ docMethod.Description = "Not available"
+ } else {
+ //purpose
+ docMethod.Description = "Not available - see request and response structs"
+
+ //describe parameters
+ docMethod.Parameters = map[string]DocParam{}
+ for i := 0; i < handler.RequestParamsType.NumField(); i++ {
+ f := handler.RequestParamsType.Field(i)
+
+ name := f.Tag.Get("json")
+ if name == "" {
+ name = f.Name
+ }
+
+ docMethod.Parameters[f.Name] = DocParam{
+ Name: name,
+ Type: fmt.Sprintf("%v", f.Type),
+ Description: f.Tag.Get("doc"),
+ }
+ }
+
+ //describe request schema
+ var err error
+ docMethod.Request, err = DocSchema(fmt.Sprintf("%s %s %s", method, path, "request"), handler.RequestBodyType)
+ if err != nil {
+ return Docs{}, errors.Wrapf(err, "failed to document request")
+ }
+ docMethod.Response, err = DocSchema(fmt.Sprintf("%s %s %s", method, path, "response"), handler.ResponseType)
+ if err != nil {
+ return Docs{}, errors.Wrapf(err, "failed to document response")
+ }
+ }
+
+ docPath.Methods[method] = docMethod
+ }
+ docs.Paths[path] = docPath
+ }
+ return docs, nil
+}
+
+func DocSchema(description string, t reflect.Type) (interface{}, error) {
+ if t == nil {
+ return nil, nil
+ }
+ schema := map[string]interface{}{
+ "description": description,
+ }
+
+ if t.Kind() == reflect.Ptr {
+ schema["optional"] = true
+ t = t.Elem()
+ }
+
+ switch t.Kind() {
+ case reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int,
+ reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint,
+ reflect.Float64, reflect.Float32,
+ reflect.Bool,
+ reflect.String:
+ schema["type"] = fmt.Sprintf("%v", t)
+
+ case reflect.Interface:
+ schema["type"] = "interface{}" //any value...?
+
+ case reflect.Struct:
+ schema["type"] = "object"
+ properties := map[string]interface{}{}
+ for i := 0; i < t.NumField(); i++ {
+ f := t.Field(i)
+ if !f.Anonymous {
+ fieldName := f.Tag.Get("json")
+ if fieldName == "" {
+ fieldName = f.Name
+ }
+ if fieldName == "-" {
+ continue //json does not marshal these
+ }
+ fieldName = strings.Replace(fieldName, ",omitempty", "", -1)
+
+ var err error
+ fieldDesc := f.Tag.Get("doc")
+ if fieldDesc == "" {
+ fieldDesc = description + "." + fieldName
+ }
+ properties[fieldName], err = DocSchema(fieldDesc, f.Type)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to document %v.%s", t, fieldName)
+ }
+ }
+ }
+ schema["properties"] = properties
+
+ case reflect.Map:
+ schema["type"] = "map"
+ keySchema, err := DocSchema("key", t.Key())
+ if err != nil {
+ return nil, errors.Wrapf(err, "cannot make schema for %v map key", t)
+ }
+ schema["key"] = keySchema
+ elemSchema, err := DocSchema("items", t.Elem())
+ if err != nil {
+ return nil, errors.Wrapf(err, "cannot make schema for %v map elem", t)
+ }
+ schema["items"] = elemSchema
+
+ case reflect.Slice:
+ schema["type"] = "array"
+ elemSchema, err := DocSchema("items", t.Elem())
+ if err != nil {
+ return nil, errors.Wrapf(err, "cannot make schema for %v slice elem", t)
+ }
+ schema["items"] = elemSchema
+
+ default:
+ return nil, errors.Errorf("cannot generate schema for %v kind=%v", t, t.Kind())
+ }
+
+ return schema, nil
+}
diff --git a/logs/api-logs.go b/api_logs/api-logs.go
similarity index 90%
rename from logs/api-logs.go
rename to api_logs/api-logs.go
index 0da4c1a..2bb51c2 100644
--- a/logs/api-logs.go
+++ b/api_logs/api-logs.go
@@ -1,4 +1,4 @@
-package logs
+package api_logs
import (
"net/url"
@@ -8,8 +8,6 @@ import (
"time"
"github.com/aws/aws-lambda-go/events"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/queues"
)
var (
@@ -30,21 +28,21 @@ func init() {
}
}
-var producer queues.Producer
+//var producer queues.Producer
-func Init(p queues.Producer) {
- producer = p
-}
+//func Init(p queues.Producer) {
+// producer = p
+//}
//Call this at the end of an API request handler to capture the req/res as well as all actions taken during the processing
//(note: action list is only reset when this is called - so must be called after each handler, else action list has to be reset at the start)
func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, req events.APIGatewayProxyRequest, res events.APIGatewayProxyResponse) error {
- if producer == nil {
- return errors.Errorf("logs queue producer not set")
- }
+ //if producer == nil {
+ // return errors.Errorf("api_logs queue producer not set")
+ //}
//todo: filter out some noisy (method+path)
- //logger.Debugf("claim: %+v", claim)
+ //logs.Debugf("claim: %+v", claim)
endTime := time.Now()
@@ -126,12 +124,12 @@ func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[stri
// }
//todo: filter out sensitive values (e.g. OTP)
- if _, err := producer.NewEvent("API_LOGS").
- Type("api-log").
- RequestID(apiLog.RequestID).
- Send(apiLog); err != nil {
- return errors.Wrapf(err, "failed to send api-log")
- }
+ //if _, err := producer.NewEvent("API_LOGS").
+ // Type("api-log").
+ // RequestID(apiLog.RequestID).
+ // Send(apiLog); err != nil {
+ // return errors.Wrapf(err, "failed to send api-log")
+ //}
return nil
} //LogIncomingAPIRequest()
@@ -139,12 +137,12 @@ func LogIncomingAPIRequest(startTime time.Time, requestID string, claim map[stri
//to capture the details
//and add it to the current handler log story for reporting/metrics
func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[string]interface{}, urlString string, method string, requestBody string, responseBody string, responseCode int) error {
- if producer == nil {
- return errors.Errorf("logs queue producer not set")
- }
+ //if producer == nil {
+ // return errors.Errorf("api_logs queue producer not set")
+ //}
//todo: filter out some noisy (method+path)
- //logger.Debugf("claim: %+v", claim)
+ //logs.Debugf("claim: %+v", claim)
endTime := time.Now()
userID, _ := claim["UserID"].(int64)
@@ -196,12 +194,12 @@ func LogOutgoingAPIRequest(startTime time.Time, requestID string, claim map[stri
}
//todo: filter out sensitive values (e.g. OTP)
- if _, err := producer.NewEvent("API_LOGS").
- Type("api-log").
- RequestID(apiLog.RequestID).
- Send(apiLog); err != nil {
- return errors.Wrapf(err, "failed to send api-log")
- }
+ //if _, err := producer.NewEvent("API_LOGS").
+ // Type("api-log").
+ // RequestID(apiLog.RequestID).
+ // Send(apiLog); err != nil {
+ // return errors.Wrapf(err, "failed to send api-log")
+ //}
return nil
} //LogOutgoingAPIRequest()
diff --git a/api_responses/api_responses.go b/api_responses/api_responses.go
new file mode 100644
index 0000000..373affe
--- /dev/null
+++ b/api_responses/api_responses.go
@@ -0,0 +1,256 @@
+package api_responses
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/utils"
+
+ "gitlab.com/uafrica/go-utils/logs"
+ "gitlab.com/uafrica/go-utils/responses"
+
+ "github.com/go-pg/pg/v10"
+
+ "github.com/aws/aws-lambda-go/events"
+)
+
+type errorMsg struct {
+ Message string `json:"message"`
+ Error string `json:"error,omitempty"`
+}
+
+// ServerError logs any error to os.Stderr and returns 500
+// Internal Server Error response that the AWS API Gateway understands.
+func ServerError(err error, msg string) (events.APIGatewayProxyResponse, error) {
+ return Error(err, msg, http.StatusInternalServerError)
+}
+
+func Error(err error, msg string, statusCode int) (events.APIGatewayProxyResponse, error) {
+ logs.ErrorWithFields(map[string]interface{}{
+ "type": "Server error",
+ "message": msg,
+ "code": statusCode,
+ }, err)
+
+ serverError := errorMsg{
+ Message: msg,
+ Error: err.Error(),
+ }
+
+ bodyBytes, err := json.Marshal(serverError)
+ if err != nil {
+ return events.APIGatewayProxyResponse{
+ StatusCode: statusCode,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Body: "{ \"error\": \"" + http.StatusText(http.StatusInternalServerError) + "\"}",
+ }, nil
+ }
+
+ return events.APIGatewayProxyResponse{
+ StatusCode: statusCode,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Body: string(bodyBytes),
+ }, errors.New(msg)
+}
+
+func DatabaseServerErrorNew(err error, msg string) error {
+ statusCode := StatusCodeFromSQLError(err)
+ errorString := err.Error()
+
+ if dbError := ErrorFromDBError(err); dbError != "" {
+ errorString = dbError
+ if strings.HasSuffix(msg, ".") {
+ // Remove trailing full stop before adding dbError.
+ msg = strings.TrimSuffix(msg, ".")
+ }
+ msg = msg + ": " + dbError
+ }
+
+ if statusCode == http.StatusNotFound {
+ logs.Info("Database error: " + msg + ". Code: " + strconv.Itoa(statusCode))
+ } else if statusCode == http.StatusConflict {
+ logs.Info("Database conflict: " + msg + ". Code: " + strconv.Itoa(statusCode))
+ } else {
+ logs.ErrorWithFields(map[string]interface{}{
+ "type": "Database error",
+ "message": msg,
+ "code": statusCode,
+ }, err)
+ }
+
+ return ServerErrorStruct{
+ error: errors.New(errorString),
+ Message: msg,
+ }
+}
+
+//implements error so that API handler can extract the msg
+type ServerErrorStruct struct {
+ error
+ Message string
+}
+
+func NewServerError(err error, msg string) error {
+ return ServerErrorStruct{
+ error: err,
+ Message: msg,
+ }
+}
+
+func DatabaseServerError(err error, msg string) (events.APIGatewayProxyResponse, error) {
+ statusCode := StatusCodeFromSQLError(err)
+ errorString := err.Error()
+
+ if dbError := ErrorFromDBError(err); dbError != "" {
+ errorString = dbError
+ if strings.HasSuffix(msg, ".") {
+ // Remove trailing full stop before adding dbError.
+ msg = strings.TrimSuffix(msg, ".")
+ }
+ msg = msg + ": " + dbError
+ }
+
+ if statusCode == http.StatusNotFound {
+ logs.Info("Database error: " + msg + ". Code: " + strconv.Itoa(statusCode))
+ } else if statusCode == http.StatusConflict {
+ logs.Info("Database conflict: " + msg + ". Code: " + strconv.Itoa(statusCode))
+ } else {
+ logs.ErrorWithFields(map[string]interface{}{
+ "type": "Database error",
+ "message": msg,
+ "code": statusCode,
+ }, err)
+ }
+
+ serverError := errorMsg{
+ Message: msg,
+ Error: errorString,
+ }
+
+ bodyBytes, marshalError := json.Marshal(serverError)
+ if marshalError != nil {
+ return events.APIGatewayProxyResponse{
+ StatusCode: statusCode,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Body: "{ \"error\": \"" + http.StatusText(http.StatusInternalServerError) + "\"}",
+ }, nil
+ }
+
+ // Don't send an error on DB conflict
+ if statusCode == http.StatusConflict {
+ err = nil
+ }
+
+ return events.APIGatewayProxyResponse{
+ StatusCode: statusCode,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Body: string(bodyBytes),
+ }, err
+}
+
+func ErrorFromDBError(err error) string {
+ pgErr, ok := err.(pg.Error)
+ if !ok {
+ return err.Error()
+ }
+
+ message := humanReadableDatabaseError(pgErr)
+ return message
+}
+
+func humanReadableDatabaseError(pgErr pg.Error) string {
+ postgresErrorCode := pgErr.Field('C')
+ if postgresErrorCode == "23505" { // Conflict
+ detail := pgErr.Field('D')
+ if detail == "" {
+ return pgErr.Error()
+ }
+
+ r, err := regexp.Compile("\\(.*?.*?\\)") // Match all between ( and )
+ if err != nil {
+ return pgErr.Error()
+ }
+
+ matches := r.FindAllString(detail, -1)
+ if len(matches) != 2 {
+ return pgErr.Error()
+ }
+
+ keysString := matches[0]
+ keysString = trimBrackets(keysString)
+ keys := strings.Split(keysString, ",")
+
+ conflictString := ""
+ for _, key := range keys {
+ cleanKey := strings.TrimSpace(key)
+ if cleanKey == "provider_id" {
+ continue // Don't check for provider ID uniqueness
+ }
+
+ cleanKey = strings.ReplaceAll(cleanKey, "_", " ")
+ if conflictString == "" {
+ conflictString = cleanKey
+ } else {
+ conflictString = conflictString + ", " + cleanKey
+ }
+ }
+
+ message := fmt.Sprintf("The specified %s already exists", conflictString)
+ return message
+ }
+
+ return pgErr.Error()
+}
+
+func trimBrackets(value string) string {
+ value = strings.TrimPrefix(value, "(")
+ value = strings.TrimSuffix(value, ")")
+ return value
+}
+
+// ClientError creates responses due to request client error
+func ClientError(status int, message string) (events.APIGatewayProxyResponse, error) {
+ logs.WarnWithFields(map[string]interface{}{
+ "type": "Client error",
+ "code": status,
+ }, errors.New(message))
+
+ e := errorMsg{
+ Message: message,
+ }
+ b, err := json.Marshal(e)
+ if err != nil {
+ logs.Info("Could not create error messsage for ", message)
+ }
+
+ return events.APIGatewayProxyResponse{
+ StatusCode: status,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), responses.ContentTypeJSONHeader),
+ Body: string(b),
+ }, errors.New(message)
+}
+
+func StatusCodeFromSQLError(err error) int {
+ if err == pg.ErrNoRows {
+ return http.StatusNotFound
+ }
+
+ pgErr, ok := err.(pg.Error)
+ if !ok || pgErr == nil || !pgErr.IntegrityViolation() {
+ return http.StatusInternalServerError
+ }
+
+ // See Postgres docs for error codes: https://www.postgresql.org/docs/10/errcodes-appendix.html
+ postgresErrorCode := pgErr.Field('C')
+ switch postgresErrorCode {
+ case "23505":
+ return http.StatusConflict
+ default:
+ return http.StatusInternalServerError
+ }
+}
diff --git a/audit/change.go b/audit/audit.go
similarity index 54%
rename from audit/change.go
rename to audit/audit.go
index e1d9649..6146f09 100644
--- a/audit/change.go
+++ b/audit/audit.go
@@ -1,72 +1,29 @@
package audit
import (
- "fmt"
"reflect"
"regexp"
- "strconv"
"strings"
- "time"
"github.com/r3labs/diff/v2"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/queues"
"gitlab.com/uafrica/go-utils/reflection"
+ "gitlab.com/uafrica/go-utils/string_utils"
)
-var producer queues.Producer
-
-func Init(p queues.Producer) {
- producer = p
-}
-
-func SaveDataChange(
- requestID string,
- source string,
- eventType string,
- orgValue interface{},
- newValue interface{},
-) error {
- if producer == nil {
- return errors.Errorf("audit queue producer not set")
- }
-
- changeRecord, err := NewChangeRecord(source, eventType, orgValue, newValue)
- if err != nil {
- return errors.Wrapf(err, "fail to determine changes")
- }
- if _, err := producer.NewEvent("AUDIT").
- Type("audit").
- RequestID(requestID).
- Send(changeRecord); err != nil {
- return errors.Wrapf(err, "failed to send data change record")
- }
- return nil
-}
-
-type ChangeRecord struct {
- ID int64 `json:"id"`
- ObjectID string `json:"object_id"`
- Type string `json:"type"`
- Source string `json:"source"`
- Timestamp time.Time `json:"timestamp"`
- Changes map[string]interface{} `json:"changes"`
+type FieldChange struct {
+ From interface{} `json:"change_from"`
+ To interface{} `json:"change_to"`
}
-//purpose:
-// Creates a record describing a change of data
-//parameters:
-// source could be "" then defaults to "SYSTEM" or specify the user name that made the change
-// orgValue and newValue could be nil
-// they are compared and changes are recorded
-func NewChangeRecord(source string, eventType string, orgValue, newValue interface{}) (ChangeRecord, error) {
- changelog, err := diff.Diff(orgValue, newValue)
+func GetChanges(original interface{}, new interface{}) (map[string]interface{}, error) {
+ changes := map[string]interface{}{}
+ changelog, err := diff.Diff(original, new)
if err != nil {
- return ChangeRecord{}, err
+ return changes, err
}
- changes := map[string]interface{}{}
for _, change := range changelog {
+
if len(change.Path) == 1 {
// Root object change
field := ToSnakeCase(change.Path[0])
@@ -103,10 +60,17 @@ func NewChangeRecord(source string, eventType string, orgValue, newValue interfa
}
} else if len(change.Path) == 3 {
+ // Array of objects
+ // ["Parcel", "0", "ActualWeight"]
+ // 0 = Object
+ // 1 = Index of object
+ // 2 = field
+
objectKey := ToSnakeCase(change.Path[0])
indexString := change.Path[1]
- index, _ := strconv.ParseInt(indexString, 10, 64)
+ index, _ := string_utils.StringToInt64(indexString)
field := ToSnakeCase(change.Path[2])
+
arrayObject, present := changes[objectKey]
if present {
if arrayOfObjects, ok := arrayObject.([]map[string]interface{}); ok {
@@ -144,41 +108,7 @@ func NewChangeRecord(source string, eventType string, orgValue, newValue interfa
}
}
- objectID := getIntValue(orgValue, "ID")
- if objectID == 0 {
- objectID = getIntValue(newValue, "ID")
- }
-
- objectIDString := fmt.Sprintf("%v", objectID)
- if objectIDString == "0" {
- objectIDString = getStringValue(orgValue, "Username")
- }
- if objectIDString == "" {
- objectIDString = getStringValue(newValue, "Username")
- }
- if objectIDString == "" {
- objectIDString = getStringValue(orgValue, "Key")
- }
- if objectIDString == "" {
- objectIDString = getStringValue(newValue, "Key")
- }
-
- event := ChangeRecord{
- ObjectID: objectIDString,
- Source: source,
- Type: eventType,
- Timestamp: time.Now(),
- Changes: changes,
- }
- if event.Source == "" {
- event.Source = "SYSTEM"
- }
- return event, nil
-}
-
-type FieldChange struct {
- From interface{} `json:"change_from"`
- To interface{} `json:"change_to"`
+ return changes, nil
}
var matchFirstCap = regexp.MustCompile("(.)([A-Z][a-z]+)")
@@ -190,7 +120,7 @@ func ToSnakeCase(str string) string {
return strings.ToLower(snake)
}
-func getIntValue(object interface{}, key string) int64 {
+func GetIntValue(object interface{}, key string) int64 {
structValue := reflect.ValueOf(object)
if structValue.Kind() == reflect.Struct {
field := structValue.FieldByName(key)
@@ -200,7 +130,7 @@ func getIntValue(object interface{}, key string) int64 {
return 0
}
-func getStringValue(object interface{}, key string) string {
+func GetStringValue(object interface{}, key string) string {
structValue := reflect.ValueOf(object)
if structValue.Kind() == reflect.Struct {
field := structValue.FieldByName(key)
diff --git a/cognito/cognito.go b/cognito/cognito.go
new file mode 100644
index 0000000..afae2c9
--- /dev/null
+++ b/cognito/cognito.go
@@ -0,0 +1,135 @@
+package cognito
+
+import (
+ "fmt"
+ "math/rand"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/logs"
+
+ "github.com/aws/aws-lambda-go/events"
+ "github.com/aws/aws-sdk-go/service/cognitoidentityprovider"
+)
+
+var CognitoService *cognitoidentityprovider.CognitoIdentityProvider
+
+// ------------------------------------------------------------------------------------------------
+// Groups
+// ------------------------------------------------------------------------------------------------
+
+func AddCognitoUserToGroup(username string, pool string, group string) error {
+ groupInput := cognitoidentityprovider.AdminAddUserToGroupInput{
+ GroupName: &group,
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ groupOutput, err := CognitoService.AdminAddUserToGroup(&groupInput)
+ logs.Info("groupOutput", groupOutput)
+ return err
+}
+
+func RemoveCognitoUserFromGroup(username string, pool string, group string) error {
+ groupInput := cognitoidentityprovider.AdminRemoveUserFromGroupInput{
+ GroupName: &group,
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ groupOutput, err := CognitoService.AdminRemoveUserFromGroup(&groupInput)
+ logs.Info("groupOutput", groupOutput)
+ return err
+}
+
+// ------------------------------------------------------------------------------------------------
+// Users
+// ------------------------------------------------------------------------------------------------
+
+func GetCognitoUser(pool string, username string) (*cognitoidentityprovider.AdminGetUserOutput, error) {
+ userInput := cognitoidentityprovider.AdminGetUserInput{
+ UserPoolId: &pool,
+ Username: &username,
+ }
+
+ userOutput, err := CognitoService.AdminGetUser(&userInput)
+ if err != nil {
+ return nil, err
+ }
+ return userOutput, nil
+}
+
+// Deletes a user from its cognito user pool
+func DeleteCognitoUser(userPoolId string, username string) error {
+ input := cognitoidentityprovider.AdminDeleteUserInput{
+ UserPoolId: &userPoolId,
+ Username: &username,
+ }
+ output, err := CognitoService.AdminDeleteUser(&input)
+ logs.Info("output", output)
+ return err
+}
+
+func ResetUserPassword(pool string, username string) (*cognitoidentityprovider.AdminResetUserPasswordOutput, error) {
+ input := cognitoidentityprovider.AdminResetUserPasswordInput{
+ UserPoolId: &pool,
+ Username: &username,
+ }
+ output, err := CognitoService.AdminResetUserPassword(&input)
+ logs.Info("output", output)
+ return output, err
+}
+
+func SetUserPassword(pool string, username string, password string) (*cognitoidentityprovider.AdminSetUserPasswordOutput, error) {
+ setPermanently := true
+ input := cognitoidentityprovider.AdminSetUserPasswordInput{
+ UserPoolId: &pool,
+ Username: &username,
+ Password: &password,
+ Permanent: &setPermanently,
+ }
+ output, err := CognitoService.AdminSetUserPassword(&input)
+ logs.Info("output", output)
+ return output, err
+}
+
+// FOR API LOGS
+
+func DetermineAuthType(identity events.APIGatewayRequestIdentity) *string {
+ result := "cognito"
+ if identity.CognitoAuthenticationType == "" {
+ result = "iam"
+ }
+
+ return &result
+}
+
+func GetAuthUsername(identity events.APIGatewayRequestIdentity) string {
+ if identity.CognitoAuthenticationProvider != "" {
+ split := strings.Split(identity.CognitoAuthenticationProvider, ":")
+ return split[len(split)-1]
+ }
+
+ // IAM
+ split := strings.Split(identity.UserArn, ":user/")
+ return split[len(split)-1]
+}
+
+// Create a pseudorandom password consisting of two three-letter words and two digits
+func RandomPassword() string {
+ i := rand.Intn(100)
+ var j int
+ for {
+ j = rand.Intn(100)
+ if j != i {
+ break
+ }
+ }
+ return fmt.Sprintf("%s%s%s", words[i], words[j], RandomDigitString(2))
+}
+
+// Create a pseudorandom string of digits (0-9) with specified length
+func RandomDigitString(len int) string {
+ var str strings.Builder
+ for i := 0; i < len; i++ {
+ fmt.Fprintf(&str, "%v", rand.Intn(10))
+ }
+ return str.String()
+}
diff --git a/cognito/words.go b/cognito/words.go
new file mode 100644
index 0000000..72c46fb
--- /dev/null
+++ b/cognito/words.go
@@ -0,0 +1,104 @@
+package cognito
+
+var words = []string{
+ "act",
+ "add",
+ "age",
+ "air",
+ "ant",
+ "ark",
+ "arm",
+ "art",
+ "ash",
+ "bad",
+ "bag",
+ "bar",
+ "bat",
+ "bay",
+ "bed",
+ "bee",
+ "big",
+ "box",
+ "bus",
+ "bye",
+ "can",
+ "car",
+ "cat",
+ "con",
+ "cow",
+ "cup",
+ "day",
+ "dog",
+ "ear",
+ "end",
+ "eye",
+ "far",
+ "fly",
+ "fox",
+ "fry",
+ "fun",
+ "gap",
+ "gym",
+ "hat",
+ "hip",
+ "hop",
+ "ice",
+ "ink",
+ "jam",
+ "jar",
+ "joy",
+ "key",
+ "kit",
+ "lab",
+ "law",
+ "log",
+ "low",
+ "man",
+ "max",
+ "may",
+ "net",
+ "nut",
+ "oil",
+ "one",
+ "out",
+ "owl",
+ "own",
+ "pen",
+ "pet",
+ "pie",
+ "pig",
+ "pin",
+ "pro",
+ "ram",
+ "rap",
+ "ray",
+ "red",
+ "row",
+ "sea",
+ "see",
+ "sew",
+ "sit",
+ "six",
+ "sky",
+ "son",
+ "spy",
+ "tan",
+ "tax",
+ "tea",
+ "ted",
+ "tee",
+ "ten",
+ "the",
+ "tin",
+ "toe",
+ "top",
+ "toy",
+ "tub",
+ "two",
+ "way",
+ "wig",
+ "yes",
+ "you",
+ "zip",
+ "zoo",
+}
diff --git a/compare/compare.go b/compare/compare.go
new file mode 100644
index 0000000..40a1f2e
--- /dev/null
+++ b/compare/compare.go
@@ -0,0 +1,17 @@
+package compare
+
+import "reflect"
+
+// PointerValuesChanged returns true when new is defined and has a different value than old
+func PointerValuesChanged(old, new interface{}) bool {
+ if new != nil {
+ if old != nil {
+ if !reflect.DeepEqual(old, new) {
+ return true
+ }
+ } else {
+ return true
+ }
+ }
+ return false
+}
diff --git a/compare/compare_test.go b/compare/compare_test.go
new file mode 100644
index 0000000..e3a6ad2
--- /dev/null
+++ b/compare/compare_test.go
@@ -0,0 +1,56 @@
+package compare
+
+import "testing"
+
+var six = 6
+var five = 5
+
+var hello = "hello"
+var world = "world"
+
+// TestPointerValuesChanges validates PointerValuesChanges functionality
+func TestPointerValuesChanges(t *testing.T) {
+
+ var tests = []struct {
+ Old interface{}
+ New interface{}
+ Expected bool
+ }{
+ {
+ Old: nil,
+ New: &six,
+ Expected: true,
+ },
+ {
+ Old: &six,
+ New: &six,
+ Expected: false,
+ },
+ {
+ Old: &five,
+ New: &six,
+ Expected: true,
+ },
+ {
+ Old: &hello,
+ New: &world,
+ Expected: true,
+ },
+ {
+ Old: &hello,
+ New: nil,
+ Expected: false,
+ },
+ {
+ Old: &world,
+ New: &world,
+ Expected: false,
+ },
+ }
+
+ for _, test := range tests {
+ if result := PointerValuesChanged(test.Old, test.New); result != test.Expected {
+ t.Errorf("Expected %v when comparing %v and %v, got %v", test.Expected, test.Old, test.New, result)
+ }
+ }
+}
diff --git a/consumer/README.md b/consumer/README.md
deleted file mode 100644
index 2d8745e..0000000
--- a/consumer/README.md
+++ /dev/null
@@ -1,4 +0,0 @@
-# Consumer
-
-Consumes a queue of events in the same way that API processes HTTP requests.
-Consumer is a type of service, just like API and CRON are also types of services.
diff --git a/consumer/check.go b/consumer/check.go
deleted file mode 100644
index 3b77c4f..0000000
--- a/consumer/check.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package consumer
-
-import "gitlab.com/uafrica/go-utils/service"
-
-type Checker interface {
- Check(service.Context) (interface{}, error)
-}
diff --git a/consumer/consumer.go b/consumer/consumer.go
deleted file mode 100644
index 6278ae9..0000000
--- a/consumer/consumer.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package consumer
-
-import "gitlab.com/uafrica/go-utils/service"
-
-//IConsumer is the interface implemented by both mem and sqs consumer
-type Consumer interface {
- WithStarter(name string, starter service.Starter) Consumer
- Run()
- ProcessFile(filename string) error
-}
diff --git a/consumer/context.go b/consumer/context.go
deleted file mode 100644
index ef182c8..0000000
--- a/consumer/context.go
+++ /dev/null
@@ -1,69 +0,0 @@
-package consumer
-
-import (
- "context"
- "encoding/json"
- "reflect"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/service"
-)
-
-//Context within a consumer to process an event
-type Context interface {
- service.Context
- Event() queues.Event //the event start started this context in the consumer
- GetRecord(recordType reflect.Type) (interface{}, error) //extract struct value from event data
-}
-
-var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
-
-type queuesContext struct {
- service.Context
- event queues.Event
-}
-
-func NewContext(service service.Service, event queues.Event) (Context, error) {
- baseCtx := context.Background()
- serviceContext, err := service.NewContext(baseCtx, event.RequestIDValue, map[string]interface{}{
- "message_type": event.TypeName,
- })
- if err != nil {
- return nil, errors.Wrapf(err, "failed to create service context")
- }
-
- ctx := queuesContext{
- Context: serviceContext,
- event: event,
- }
- return ctx, nil
-}
-
-func (ctx queuesContext) Event() queues.Event {
- return ctx.event
-}
-
-func (ctx queuesContext) RequestID() string {
- return ctx.event.RequestIDValue
-}
-
-func (ctx queuesContext) GetRecord(recordType reflect.Type) (interface{}, error) {
- recordValuePtr := reflect.New(recordType)
- err := json.Unmarshal([]byte(ctx.event.BodyJSON), recordValuePtr.Interface())
- if err != nil {
- return nil, errors.Wrapf(err, "failed to JSON decode message body")
- }
-
- if validator, ok := recordValuePtr.Interface().(IValidator); ok {
- if err := validator.Validate(); err != nil {
- return nil, errors.Wrapf(err, "invalid message body")
- }
- }
-
- return recordValuePtr.Elem().Interface(), nil
-}
-
-type IValidator interface {
- Validate() error
-}
diff --git a/consumer/handler.go b/consumer/handler.go
deleted file mode 100644
index bc09bd7..0000000
--- a/consumer/handler.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package consumer
-
-import (
- "reflect"
-
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type Handler struct {
- RecordType reflect.Type
- FuncValue reflect.Value
-}
-
-func NewHandler(fnc interface{}) (Handler, error) {
- h := Handler{}
-
- fncType := reflect.TypeOf(fnc)
- if fncType.NumIn() != 2 {
- return h, errors.Errorf("takes %d args instead of (Context, Record)", fncType.NumIn())
- }
- if fncType.NumOut() != 1 {
- return h, errors.Errorf("returns %d results instead of (error)", fncType.NumOut())
- }
-
- //arg[0] must implement interface sqs.IContext
- if fncType.In(0) != contextInterfaceType &&
- !fncType.In(0).Implements(contextInterfaceType) {
- return h, errors.Errorf("first arg %v does not implement %v", fncType.In(0), contextInterfaceType)
- }
-
- //arg[1] must be a struct for the message record body. It may be an empty struct, but
- //all public fields require a json tag which we will use to math the URL param name
- if err := validateStructType(fncType.In(1)); err != nil {
- return h, errors.Errorf("second arg %v is not valid record struct type", fncType.In(1))
- }
- h.RecordType = fncType.In(1)
-
- //result must be error
- if _, ok := reflect.New(fncType.Out(0)).Interface().(*error); !ok {
- return h, errors.Errorf("result %v is not error type", fncType.Out(0))
- }
-
- h.FuncValue = reflect.ValueOf(fnc)
- return h, nil
-}
-
-func validateStructType(t reflect.Type) error {
- if t.Kind() != reflect.Struct {
- return errors.Errorf("%v is %v, not a struct", t, t.Kind())
- }
- return nil
-}
diff --git a/consumer/mem_consumer/README.md b/consumer/mem_consumer/README.md
deleted file mode 100644
index 58ae44f..0000000
--- a/consumer/mem_consumer/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# Memory Queues
-
-This is an in-memory implementation of go-utils/queues for use in local development and testing only.
\ No newline at end of file
diff --git a/consumer/mem_consumer/consumer.go b/consumer/mem_consumer/consumer.go
deleted file mode 100644
index 37d903a..0000000
--- a/consumer/mem_consumer/consumer.go
+++ /dev/null
@@ -1,186 +0,0 @@
-package mem_consumer
-
-import (
- "encoding/json"
- "fmt"
- "math/rand"
- "os"
- "reflect"
- "sync"
- "time"
-
- "github.com/google/uuid"
- "gitlab.com/uafrica/go-utils/consumer"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/service"
-)
-
-func New(s service.Service, routes map[string]interface{}) consumer.Consumer {
- if s == nil {
- panic("NewConsumer(service==nil)")
- }
- router, err := consumer.NewRouter(routes)
- if err != nil {
- panic(fmt.Sprintf("cannot create router: %+v", err))
- }
- c := &memConsumer{
- Service: s,
- router: router,
- queues: map[string]*queue{},
- }
-
- //create a producer that will produce into this consumer
- c.producer = &memProducer{
- consumer: c,
- }
- c.Service = c.Service.WithProducer(c.producer)
- return c
-}
-
-type memConsumer struct {
- sync.Mutex
- service.Service
- router consumer.Router
- producer queues.Producer
- queues map[string]*queue
-}
-
-//wrap Service.WithStarter to return cron, else cannot be chained
-func (consumer *memConsumer) WithStarter(name string, starter service.Starter) consumer.Consumer {
- consumer.Service = consumer.Service.WithStarter(name, starter)
- return consumer
-}
-
-func (consumer *memConsumer) Queue(name string) (*queue, error) {
- consumer.Lock()
- defer consumer.Unlock()
- q, ok := consumer.queues[name]
- if !ok {
- q = &queue{
- consumer: consumer,
- name: name,
- ch: make(chan queues.Event),
- }
- go q.run()
- consumer.queues[name] = q
- }
- return q, nil
-}
-
-//do not call this - when using local producer, the consumer is automatically running
-//for each queue you send to, and processing from q.run()
-func (consumer *memConsumer) Run() {
- panic(errors.Errorf("DO NOT RUN LOCAL CONSUMER"))
-}
-
-func (consumer *memConsumer) ProcessFile(filename string) error {
- f, err := os.Open(filename)
- if err != nil {
- return errors.Wrapf(err, "failed to open queue event file %s", filename)
- }
- defer f.Close()
-
- var event queues.Event
- if err := json.NewDecoder(f).Decode(&event); err != nil {
- return errors.Wrapf(err, "failed to read queues.Event from file %s", filename)
- }
-
- q := queue{
- consumer: consumer,
- name: "NoName",
- ch: nil,
- }
-
- if q.process(
- event,
- ); err != nil {
- return errors.Wrapf(err, "failed to process event from file %s", filename)
- }
- return nil
-}
-
-type queue struct {
- consumer *memConsumer
- name string
- ch chan queues.Event
-}
-
-func (q *queue) run() {
- // logger.Debugf("Q(%s) Start", q.name)
- for event := range q.ch {
- //process in background because some event processing sends to itself then wait for some responses on new events on the same queue!!!
- go func(event queues.Event) {
- // logger.Debugf("Q(%s) process start: %+v", q.name, event)
- err := q.process(event)
- if err != nil {
- q.consumer.Errorf("Q(%s) process failed: %+v", q.name, err)
- // } else {
- // q.consumer.Debugf("Q(%s) process success: %+v", q.name, err)
- }
- }(event)
- }
- // logger.Debugf("Q(%s) STOPPED", q.name)
-}
-
-func (q *queue) process(event queues.Event) error {
- //todo: create context with logger
- rand.Seed(time.Now().Unix())
-
- //report handler crashes
- // if q.crashReporter != nil {
- // defer q.crashReporter.Catch(ctx)
- // }
- ctx, err := consumer.NewContext(q.consumer.Service, event)
- if err != nil {
- return err
- }
-
- //routing on messageType
- sqsHandler, err := q.consumer.router.Route(event.TypeName)
- if err != nil {
- return errors.Wrapf(err, "unhandled event type(%v)", event.TypeName)
- }
- handler, ok := sqsHandler.(consumer.Handler)
- if !ok {
- return errors.Errorf("messageType(%v) unsupported signature: %T", event.TypeName, sqsHandler)
- }
-
- args := []reflect.Value{
- reflect.ValueOf(ctx),
- }
-
- //allocate, populate and validate request struct
- var recordStruct interface{}
- recordStruct, err = ctx.GetRecord(handler.RecordType)
- if err != nil {
- return errors.Wrapf(err, "invalid message body")
- }
-
- //log if not internal queue
- if q.name != "AUDIT" && q.name != "API_LOGS" {
- ctx.WithFields(map[string]interface{}{
- "params": event.ParamValues,
- "body": event.BodyJSON,
- }).Infof("RECV(%s) Queue(%s).Type(%s).Due(%s)",
- "---", //not yet available here - not part of event, and in SQS I think it is passed in SQS layer, so need to extend local channel to include this along with event
- q.name,
- event.TypeName,
- event.DueTime)
- ctx.Tracef("RECV(%s) Request(%T)%v", q.name, recordStruct, recordStruct)
- }
- args = append(args, reflect.ValueOf(recordStruct))
-
- results := handler.FuncValue.Call(args)
- if len(results) > 0 && !results[0].IsNil() {
- return errors.Wrapf(results[0].Interface().(error), "handler failed")
- }
- ctx.Debugf("Handler done")
- return nil
-} //queue.process()
-
-func (q *queue) Send(event queues.Event) (msgID string, err error) {
- event.MessageID = uuid.New().String()
- q.ch <- event
- return event.MessageID, nil
-}
diff --git a/consumer/mem_consumer/producer.go b/consumer/mem_consumer/producer.go
deleted file mode 100644
index f1b51f0..0000000
--- a/consumer/mem_consumer/producer.go
+++ /dev/null
@@ -1,44 +0,0 @@
-package mem_consumer
-
-import (
- "gitlab.com/uafrica/go-utils/consumer"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
-)
-
-//can only produce locally if also consuming local
-func NewProducer(consumer consumer.Consumer) queues.Producer {
- if consumer == nil {
- panic(errors.Errorf("cannot produce local events without mem consumer"))
- }
- mc, ok := consumer.(*memConsumer)
- if !ok {
- panic(errors.Errorf("NewProducer(consumer=%T) is not a mem consumer", consumer))
- }
- return mc.producer
-}
-
-type memProducer struct {
- consumer *memConsumer
-}
-
-func (producer *memProducer) Send(event queues.Event) (string, error) {
- logger.Debugf("MEM producer.queue(%s) Sending event %+v", event.QueueName, event)
- q, err := producer.consumer.Queue(event.QueueName)
- if err != nil {
- return "", errors.Wrapf(err, "failed to get/create queue(%s)", event.QueueName)
- }
-
- msgID, err := q.Send(event)
- if err != nil {
- return "", errors.Wrapf(err, "failed to send to queue(%s)", event.QueueName)
- }
-
- logger.Debugf("MEM producer.queue(%s) SENT event %+v", event.QueueName, event)
- return msgID, nil
-}
-
-func (producer *memProducer) NewEvent(queueName string) queues.Event {
- return queues.NewEvent(producer, queueName)
-}
diff --git a/consumer/router.go b/consumer/router.go
deleted file mode 100644
index 744cf1a..0000000
--- a/consumer/router.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package consumer
-
-import (
- "fmt"
-
- "github.com/aws/aws-lambda-go/events"
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type Router struct {
- endpoints map[string]interface{}
-}
-
-func (r Router) Endpoints() map[string]interface{} {
- return r.endpoints
-}
-
-func (r Router) Route(messageType string) (interface{}, error) {
- if handler, ok := r.endpoints[messageType]; !ok {
- return nil, errors.Errorf("%s not found", messageType)
- } else {
- return handler, nil
- }
-}
-
-//check that all endpoints are correctly defined using one of the supported handler types
-//return updated endpoints with additional information
-func NewRouter(endpoints map[string]interface{}) (Router, error) {
- countLegacyEvent := 0
- countLegacyMessage := 0
- countHandler := 0
- for messageType, handlerFunc := range endpoints {
- if messageType == "" {
- return Router{}, errors.Errorf("blank messageType")
- }
- if messageType == "/sqs-docs" {
- return Router{}, errors.Errorf("%s may not be a defined endpoint - it is reserved", messageType)
- }
- if handlerFunc == nil {
- return Router{}, errors.Errorf("nil handler on %s", messageType)
- }
-
- if _, ok := handlerFunc.(func(event events.SQSEvent) error); ok {
- //ok - leave as is - we support this legacyHandler (typical in shiplogic)
- fmt.Printf("%30.30s: OK (legacy event handler)\n", messageType)
- countLegacyEvent++
- } else {
- handler, err := NewHandler(handlerFunc)
- if err != nil {
- return Router{}, errors.Wrapf(err, "%30.30s has invalid handler %T", messageType, handlerFunc)
- }
-
- //replace the endpoint value so we can quickly call this handler
- endpoints[messageType] = handler
- fmt.Printf("%30.30s: OK (record: %v)\n", messageType, handler.RecordType)
- countHandler++
- }
- }
- fmt.Printf("Checked %d legacy event and %d legacy message and %d new handlers\n", countLegacyEvent, countLegacyMessage, countHandler)
-
- //add reserved endpoint to generate documentation
- r := Router{
- endpoints: endpoints,
- }
-
- // {
- // docsHandler, err := NewHandler(GETApiDocs(r)) //endpoints))
- // if err != nil {
- // return Router{}, errors.Wrapf(err, "failed to define handler for docs")
- // }
- // endpoints["/api-docs"] = map[string]interface{}{
- // "GET": docsHandler,
- // }
- // }
- return r, nil
-}
diff --git a/consumer/sqs_consumer/consumer.go b/consumer/sqs_consumer/consumer.go
deleted file mode 100644
index b34eb6f..0000000
--- a/consumer/sqs_consumer/consumer.go
+++ /dev/null
@@ -1,210 +0,0 @@
-package sqs_consumer
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "math/rand"
- "os"
- "path"
- "reflect"
- "strings"
- "time"
-
- "github.com/aws/aws-lambda-go/events"
- "github.com/aws/aws-lambda-go/lambda"
- "github.com/aws/aws-lambda-go/lambdacontext"
- "github.com/google/uuid"
- "gitlab.com/uafrica/go-utils/audit"
- "gitlab.com/uafrica/go-utils/consumer"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logs"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/queues/sqs_producer"
- "gitlab.com/uafrica/go-utils/service"
-)
-
-func New(requestIDHeaderKey string, routes map[string]interface{}) consumer.Consumer {
- env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
- if env == "" {
- env = "dev"
- }
- router, err := consumer.NewRouter(routes)
- if err != nil {
- panic(fmt.Sprintf("cannot create router: %+v", err))
- }
-
- //legacy message type - when running SQS instance for one type of messages only
- //when defined, make sure handler exists for this type
- sqsMessageType := os.Getenv("SQS_MESSAGE_TYPE")
- if sqsMessageType != "" {
- if _, err := router.Route(sqsMessageType); err != nil {
- panic(errors.Errorf("No route defined for SQS_MESSAGE_TYPE=\"%s\"", sqsMessageType))
- }
- }
-
- producer := sqs_producer.New(requestIDHeaderKey)
- s := service.New().
- WithProducer(producer)
- audit.Init(producer)
- logs.Init(producer)
-
- return sqsConsumer{
- Service: s,
- env: env,
- router: router,
- requestIDHeaderKey: requestIDHeaderKey,
- ConstantMessageType: sqsMessageType,
- checks: map[string]consumer.Checker{},
- }
-}
-
-type sqsConsumer struct {
- service.Service
- env string
- router consumer.Router
- requestIDHeaderKey string
- ConstantMessageType string //from os.Getenv("SQS_MESSAGE_TYPE")
- checks map[string]consumer.Checker
-}
-
-//wrap Service.WithStarter to return cron, else cannot be chained
-func (c sqsConsumer) WithStarter(name string, starter service.Starter) consumer.Consumer {
- c.Service = c.Service.WithStarter(name, starter)
- return c
-}
-
-func (c sqsConsumer) Run() {
- lambda.Start(c.Handler)
-}
-
-func (c sqsConsumer) ProcessFile(filename string) error {
- f, err := os.Open(filename)
- if err != nil {
- return errors.Wrapf(err, "failed to open queue event file %s", filename)
- }
- defer f.Close()
-
- var event events.SQSEvent
- if err := json.NewDecoder(f).Decode(&event); err != nil {
- return errors.Wrapf(err, "failed to read sqs event from file %s", filename)
- }
-
- if c.Handler(
- lambdacontext.NewContext(
- context.Background(),
- &lambdacontext.LambdaContext{
- AwsRequestID: uuid.New().String(),
- InvokedFunctionArn: strings.TrimSuffix(path.Base(filename), ".json"),
- // Identity CognitoIdentity
- // ClientContext ClientContext
- },
- ),
- event,
- ); err != nil {
- return errors.Wrapf(err, "failed to process event from file %s", filename)
- }
- return nil
-}
-
-func (c sqsConsumer) Handler(baseCtx context.Context, lambdaEvent events.SQSEvent) error {
- //todo: create context with logger
- rand.Seed(time.Now().Unix())
-
- //report handler crashes
- // if consumer.crashReporter != nil {
- // defer sqs.crashReporter.Catch(ctx)
- // }
-
- if c.ConstantMessageType != "" {
- //legacy mode for fixed message type as used in shiplogic
- //where the whole instance is started for a specific SQS_MESSAGE_TYPE defined in environment
- handler, err := c.router.Route(c.ConstantMessageType)
- if err != nil {
- return errors.Wrapf(err, "messageType=%s not handled", c.ConstantMessageType) //checked on startup - should never get here!!!
- }
-
- if msgHandler, ok := handler.(func(events.SQSEvent) error); !ok {
- return errors.Wrapf(err, "SQS_MESSAGE_TYPE=%s: handler signature %T not supported", c.ConstantMessageType, handler)
- } else {
- //call the handler
- return msgHandler(lambdaEvent)
- }
- } else {
- //support different message types - obtained from the individual event records
- //process all message records in this event:
- for messageIndex, message := range lambdaEvent.Records {
- //get request-id for this message record
- requestID := ""
- if requestIDAttr, ok := message.MessageAttributes[c.requestIDHeaderKey]; ok {
- requestID = *requestIDAttr.StringValue
- }
-
- messageType := ""
- var handlerErr error
- if messageTypeAttr, ok := message.MessageAttributes["type"]; !ok || messageTypeAttr.StringValue == nil {
- c.Errorf("ignoring message without messageType") //todo: could support generic handler for these... not yet required
- continue
- } else {
- messageType = *messageTypeAttr.StringValue
- }
-
- event := queues.Event{
- //producer: nil,
- MessageID: message.MessageId,
- QueueName: "N/A", //not sure how to get queue name from lambda Event... would be good to log it, may be in os.Getenv(???)?
- TypeName: messageType,
- DueTime: time.Now(),
- RequestIDValue: requestID,
- BodyJSON: message.Body,
- }
-
- ctx, err := consumer.NewContext(c.Service, event)
- if err != nil {
- return err
- }
-
- //log if not internal queue
- if ctx.Event().QueueName != "AUDIT" && ctx.Event().QueueName != "API_LOGS" {
- ctx.WithFields(map[string]interface{}{
- "message_index": messageIndex,
- "message": message,
- }).Infof("Queue(%s) Start SQS Handler Event: %v", ctx.Event().QueueName, ctx.Event())
- }
-
- //routing on messageType
- sqsHandler, err := c.router.Route(messageType)
- if err != nil {
- ctx.Errorf("Unhandled sqs messageType(%v): %v", messageType, err)
- continue
- }
- handler, ok := sqsHandler.(consumer.Handler)
- if !ok {
- ctx.Errorf("messageType(%v) unsupported signature: %T", messageType, sqsHandler)
- continue
- }
-
- args := []reflect.Value{
- reflect.ValueOf(ctx),
- }
-
- //allocate, populate and validate request struct
- var recordStruct interface{}
- recordStruct, err = ctx.GetRecord(handler.RecordType)
- if err != nil {
- ctx.Errorf("invalid message: %+v", err)
- continue
- }
-
- ctx.Tracef("message (%T) %+v", recordStruct, recordStruct)
- args = append(args, reflect.ValueOf(recordStruct))
-
- results := handler.FuncValue.Call(args)
- if len(results) > 0 && !results[0].IsNil() {
- handlerErr = results[0].Interface().(error)
- ctx.Errorf("handler failed: %+v", handlerErr)
- }
- }
- }
- return nil
-}
diff --git a/cron/check.go b/cron/check.go
deleted file mode 100644
index 71a3e53..0000000
--- a/cron/check.go
+++ /dev/null
@@ -1,5 +0,0 @@
-package cron
-
-type ICheck interface {
- Check(Context) (interface{}, error)
-}
diff --git a/cron/context.go b/cron/context.go
deleted file mode 100644
index 7903e70..0000000
--- a/cron/context.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package cron
-
-import (
- "context"
- "reflect"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/service"
-)
-
-type Context interface {
- service.Context
-}
-
-var contextInterfaceType = reflect.TypeOf((*Context)(nil)).Elem()
-
-type cronContext struct {
- service.Context
- name string //cron function name
-}
-
-func (cron Cron) NewContext(baseCtx context.Context, requestID string, cronName string) (Context, error) {
- serviceContext, err := cron.Service.NewContext(baseCtx, requestID, map[string]interface{}{
- "cron": cronName,
- })
- if err != nil {
- return nil, errors.Wrapf(err, "failed to create service context")
- }
-
- ctx := cronContext{
- Context: serviceContext,
- name: cronName,
- }
- return ctx, nil
-}
diff --git a/cron/cron.go b/cron/cron.go
deleted file mode 100644
index 114e106..0000000
--- a/cron/cron.go
+++ /dev/null
@@ -1,122 +0,0 @@
-package cron
-
-import (
- "context"
- "fmt"
- "os"
-
- "github.com/aws/aws-lambda-go/lambda"
- "github.com/aws/aws-lambda-go/lambdacontext"
- "github.com/google/uuid"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/service"
- "gitlab.com/uafrica/go-utils/string_utils"
-)
-
-func New(functions map[string]func(Context) error) Cron {
- env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
- if env == "" {
- env = "dev"
- }
-
- router, err := NewRouter(functions)
- if err != nil {
- panic(fmt.Sprintf("cannot create router: %+v", err))
- }
-
- return Cron{
- Service: service.New(),
- env: env,
- router: router,
- checks: map[string]ICheck{},
- crashReporter: defaultCrashReporter{},
- }
-}
-
-type Cron struct {
- service.Service
- env string
- router Router
- checks map[string]ICheck
- crashReporter ICrashReporter
-}
-
-//wrap Service.WithStarter to return cron, else cannot be chained
-func (cron Cron) WithStarter(name string, starter service.Starter) Cron {
- cron.Service = cron.Service.WithStarter(name, starter)
- return cron
-}
-
-//add a check to startup of each context
-//they will be called in the sequence they were added
-//if check return error, processing stops and err is returned
-//if check succeed, and return !=nil data, it is stored against the name
-// so your handler can retieve it with:
-// checkData := ctx.Value(name).(expectedType)
-// or
-// checkData,ok := ctx.Value(name).(expectedType)
-// if !ok { ... }
-//you can implement one check that does everything and return a struct or
-//implement one for your db, one for rate limit, one for auth, one for ...
-//the name must be snake-case, e.g. "this_is_my_check_data_name"
-func (cron Cron) WithCheck(name string, check ICheck) Cron {
- if !string_utils.IsSnakeCase(name) {
- panic(errors.Errorf("invalid check name=\"%s\", expecting snake_case names only", name))
- }
- if check == nil {
- panic(errors.Errorf("check(%s) func==nil", name))
- }
- if _, ok := cron.checks[name]; ok {
- panic(errors.Errorf("check(%s) already defined", name))
- }
- cron.checks[name] = check
- return cron
-}
-
-func (cron Cron) WithCrashReporter(crashReporter ICrashReporter) Cron {
- if crashReporter != nil {
- cron.crashReporter = crashReporter
- }
- return cron
-}
-
-type ICrashReporter interface {
- Catch(ctx Context) //Report(method string, path string, crash interface{})
-}
-
-type defaultCrashReporter struct{}
-
-func (defaultCrashReporter) Catch(ctx Context) {
- // crash := recover()
- // if crash != nil {
- // ctx.Errorf("CRASH: (%T) %+v\n", crash, crash)
- // }
-}
-
-func (cron Cron) Run(invokeArn *string) {
- if invokeArn != nil && *invokeArn != "" {
- //just run this handler and terminate - for testing on a terminal
- logger.Infof("Invoking ARN=%s locally for testing ...", *invokeArn)
-
- lambdaContext := lambdacontext.NewContext(
- context.Background(),
- &lambdacontext.LambdaContext{
- AwsRequestID: uuid.New().String(),
- InvokedFunctionArn: *invokeArn,
- // Identity CognitoIdentity
- // ClientContext ClientContext
- },
- )
- err := cron.Handler(lambdaContext)
- if err != nil {
- logger.Errorf("local cron handler failed: %+v", err)
- } else {
- logger.Debugf("local cron success")
- }
- return
- }
-
- //production
- lambda.Start(cron.Handler)
-}
diff --git a/cron/handler.go b/cron/handler.go
deleted file mode 100644
index 814a41e..0000000
--- a/cron/handler.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package cron
-
-import (
- "reflect"
-
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type Handler struct {
- RecordType reflect.Type
- FuncValue reflect.Value
-}
-
-func NewHandler(fnc interface{}) (Handler, error) {
- h := Handler{}
-
- fncType := reflect.TypeOf(fnc)
- if fncType.NumIn() != 2 {
- return h, errors.Errorf("takes %d args instead of (Context, Record)", fncType.NumIn())
- }
- if fncType.NumOut() != 1 {
- return h, errors.Errorf("returns %d results instead of (error)", fncType.NumOut())
- }
-
- //arg[0] must implement interface consumer.Context
- if fncType.In(0) != contextInterfaceType &&
- !fncType.In(0).Implements(contextInterfaceType) {
- return h, errors.Errorf("first arg %v does not implement %v", fncType.In(0), contextInterfaceType)
- }
-
- //arg[1] must be a struct for the message record body. It may be an empty struct, but
- //all public fields require a json tag which we will use to math the URL param name
- if err := validateStructType(fncType.In(1)); err != nil {
- return h, errors.Errorf("second arg %v is not valid record struct type", fncType.In(1))
- }
- h.RecordType = fncType.In(1)
-
- //result must be error
- if _, ok := reflect.New(fncType.Out(0)).Interface().(*error); !ok {
- return h, errors.Errorf("result %v is not error type", fncType.Out(0))
- }
-
- h.FuncValue = reflect.ValueOf(fnc)
- return h, nil
-}
-
-func validateStructType(t reflect.Type) error {
- if t.Kind() != reflect.Struct {
- return errors.Errorf("%v is %v, not a struct", t, t.Kind())
- }
- return nil
-}
diff --git a/cron/lambda.go b/cron/lambda.go
deleted file mode 100644
index 3043aaa..0000000
--- a/cron/lambda.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package cron
-
-import (
- "context"
- "math/rand"
- "os"
- "time"
-
- "github.com/aws/aws-lambda-go/lambdacontext"
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type LambdaCronHandler func(lambdaCtx context.Context) error
-
-func (cron Cron) Handler(lambdaCtx context.Context) (err error) {
- lc, _ := lambdacontext.FromContext(lambdaCtx)
- requestID := lc.AwsRequestID
-
- //if env CRON_MESSAGE_TYPE is defined, this instance only process that type
- //if not, try to use ARN from the event context which allows one instance to
- //call any handler
- routeKey := os.Getenv("CRON_MESSAGE_TYPE")
- if routeKey == "" {
- routeKey = lc.InvokedFunctionArn
- }
-
- cronName, cronFunc := cron.router.Route(routeKey)
- if cronFunc == nil {
- return errors.Errorf("request-id:%s unknown cron function(%s)", requestID, routeKey)
- }
-
- //got a handler, prepare to run:
- rand.Seed(time.Now().Unix())
-
- //service context invoke the starters and could fail, e.g. if cannot connect to db
- ctx, err := cron.NewContext(lambdaCtx, requestID, cronName)
- if err != nil {
- return err
- }
-
- defer func() {
- if err != nil {
- ctx.Errorf("failed: %+v", err)
- }
- }()
-
- //report handler crashes
- if cron.crashReporter != nil {
- defer cron.crashReporter.Catch(ctx)
- }
-
- for checkName, check := range cron.checks {
- var checkData interface{}
- checkData, err = check.Check(ctx)
- if err != nil {
- err = errors.Wrapf(err, "%s", checkName)
- return
- }
- if err = ctx.Set(checkName, checkData); err != nil {
- err = errors.Wrapf(err, "failed to set check(%s) data=(%T)%+v", checkName, checkData, checkData)
- return
- }
- }
-
- //todo: set log level, trigger log on conditions, sync at end of transaction - after log level was determined
- ctx.Infof("Start CRON Handler")
-
- if err := cronFunc(ctx); err != nil {
- return errors.Wrapf(err, "Cron(%s) failed", cronName)
- }
- return nil
-}
diff --git a/cron/router.go b/cron/router.go
deleted file mode 100644
index c58bf7f..0000000
--- a/cron/router.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package cron
-
-import (
- "fmt"
- "strings"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-type Router struct {
- endpoints map[string]func(Context) error
-}
-
-func (r Router) Endpoints() map[string]func(Context) error {
- return r.endpoints
-}
-
-func (r Router) Route(arn string) (string, func(Context) error) {
- for name, hdlr := range r.endpoints {
- if strings.Contains(arn, name) {
- return name, hdlr
- } else {
- logger.Debugf("ARN(%s) does not contain cronName(%s)", arn, name)
- }
- }
- return "", nil
-}
-
-//check that all endpoints are correctly defined using one of the supported handler types
-//return updated endpoints with additional information
-func NewRouter(endpoints map[string]func(Context) error) (Router, error) {
- countLegacyEvent := 0
- countLegacyMessage := 0
- countHandler := 0
- for messageType, handlerFunc := range endpoints {
- if messageType == "" {
- return Router{}, errors.Errorf("blank messageType")
- }
- if messageType == "/sqs-docs" {
- return Router{}, errors.Errorf("%s may not be a defined endpoint - it is reserved", messageType)
- }
- if handlerFunc == nil {
- return Router{}, errors.Errorf("nil handler on %s", messageType)
- }
- fmt.Printf("%30.30s: OK\n", messageType)
- }
- fmt.Printf("Checked %d legacy event and %d legacy message and %d new handlers\n", countLegacyEvent, countLegacyMessage, countHandler)
-
- //add reserved endpoint to generate documentation
- r := Router{
- endpoints: endpoints,
- }
-
- // {
- // docsHandler, err := NewHandler(GETApiDocs(r)) //endpoints))
- // if err != nil {
- // return Router{}, errors.Wrapf(err, "failed to define handler for docs")
- // }
- // endpoints["/api-docs"] = map[string]interface{}{
- // "GET": docsHandler,
- // }
- // }
- return r, nil
-}
diff --git a/date_utils/date_utils.go b/date_utils/date_utils.go
new file mode 100644
index 0000000..2667554
--- /dev/null
+++ b/date_utils/date_utils.go
@@ -0,0 +1,118 @@
+package date_utils
+
+import (
+ "strconv"
+ "time"
+)
+
+const TimeZoneString = "Africa/Johannesburg"
+
+func DateLayoutYearMonthDayTimeT() string {
+ layout := "2006-01-02T15:04:05"
+ return layout
+}
+
+func DateLayoutYearMonthDayTimeTZ() string {
+ layout := "2006-01-02T15:04:05Z"
+ return layout
+}
+
+func DateLayoutYearMonthDayTimeTimezone() string {
+ layout := "2006-01-02 15:04:05-07:00"
+ return layout
+}
+
+func DateLayoutForDB() string {
+ layout := "2006-01-02 15:04:05-07"
+ return layout
+}
+
+func DateLayoutYearMonthDayTime() string {
+ layout := "2006-01-02 15:04:05"
+ return layout
+}
+
+func DateLayoutFilenameSafe() string {
+ layout := "2006-01-02(15h04s05)"
+ return layout
+}
+
+func DateLayoutYearMonthDay() string {
+ layout := "2006-01-02"
+ return layout
+}
+
+func DateLayoutTime() string {
+ layout := "15:04:05"
+ return layout
+}
+
+func DateLayoutHumanReadable() string {
+ layout := "02 Jan 2006"
+ return layout
+}
+
+func DateLayoutTrimmed() string {
+ layout := "20060102150405"
+ return layout
+}
+
+func DateDBFormattedString(date time.Time) string {
+ return date.Format("2006-01-02 15:04:05.000000-07")
+}
+
+func DateDBFormattedStringDateOnly(date time.Time) string {
+ return date.Format("2006-01-02")
+}
+
+func CurrentLocation() *time.Location {
+ loc, _ := time.LoadLocation(TimeZoneString)
+ return loc
+}
+
+func DateLocal(date *time.Time) {
+ if date == nil {
+ return
+ }
+ *date = (*date).In(CurrentLocation())
+}
+
+func CurrentDate() time.Time {
+ currentDate := time.Now().In(CurrentLocation())
+ return currentDate
+}
+
+func DateEqual(date1, date2 time.Time) bool {
+ y1, m1, d1 := date1.Date()
+ y2, m2, d2 := date2.Date()
+ return y1 == y2 && m1 == m2 && d1 == d2
+}
+
+// TimeBefore determines whether a (string format HH:mm) is earlier than b (string format HH:mm)
+func TimeBefore(a string, b string) bool {
+ if len(a) < 5 || len(b) < 5 {
+ return false // can't detemrine before/after
+ }
+
+ hoursA, _ := strconv.Atoi(a[0:2])
+ hoursB, _ := strconv.Atoi(b[0:2])
+
+ minA, _ := strconv.Atoi(a[3:5])
+ minB, _ := strconv.Atoi(b[3:5])
+
+ if hoursA == hoursB {
+ return minA < minB
+ }
+
+ return hoursA < hoursB
+}
+
+// ConvertToNoDateTimeString - Converts a PSQL Time type to Go Time type
+func ConvertToNoDateTimeString(timeString *string) (*string, error) {
+ parsedTime, err := time.Parse("15:04:05", *timeString)
+ if err != nil {
+ return nil, err
+ }
+ formattedTime := parsedTime.Format("15:04")
+ return &formattedTime, nil
+}
diff --git a/encryption/encryption.go b/encryption/encryption.go
new file mode 100644
index 0000000..fd5959d
--- /dev/null
+++ b/encryption/encryption.go
@@ -0,0 +1,22 @@
+package encryption
+
+import (
+ "crypto/hmac"
+ "crypto/md5"
+ "crypto/sha256"
+ "encoding/base64"
+ "fmt"
+)
+
+func Hash(input string, key string) string {
+ keyBytes := []byte(key)
+ h := hmac.New(sha256.New, keyBytes)
+ h.Write([]byte(input))
+ return base64.StdEncoding.EncodeToString(h.Sum(nil))
+}
+
+func Md5HashString(bytesToHash []byte) string {
+ hash := md5.Sum(bytesToHash)
+ hashString := fmt.Sprintf("%X", hash)
+ return hashString
+}
diff --git a/examples/core/api/main.go b/examples/core/api/main.go
deleted file mode 100644
index 5fe8aa1..0000000
--- a/examples/core/api/main.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package main
-
-import (
- "math/rand"
- "net/http"
- "os"
-
- "gitlab.com/uafrica/go-utils/api"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/examples/core/app"
- "gitlab.com/uafrica/go-utils/examples/core/db"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-func main() {
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.SetGlobalFormat(logger.NewConsole())
- api.New("uafrica-request-id", app.ApiRoutes()).
- WithStarter("db", db.Connector("core")).
- WithCheck("claims", claimsChecker{}).
- WithCheck("maintenance", maint{}).
- WithCheck("rate", rateLimiter{}).
- WithCORS(cors{}).
- WithEvents(app.QueueRoutes()). //only used when LOG_LEVEL="debug"
- Run()
-}
-
-type claimsChecker struct{}
-
-type Claims struct {
- AccountID int64
- UserID int64
-}
-
-func (claimsChecker) Check(ctx api.Context) (interface{}, error) {
- //then extract auth claim and check against the db ...
- claims := Claims{
- UserID: 1,
- AccountID: 13,
- }
-
- //set it in the API context (can be retrieved with api.Context.ClaimGet())
- ctx.ClaimSet("AccountID", claims.AccountID)
- ctx.ClaimSet("UserID", claims.UserID)
-
- //return the struct (can be retrieved with service.Context.Get()/Value())
- return claims, nil
-}
-
-type maint struct{}
-
-//for maintenance mode, put a message in environment variable MAINTENANCE_MODE
-//then than message will be displayed in the response. Clear the variable to
-//proceed normal operation
-func (m maint) Check(ctx api.Context) (interface{}, error) {
- msg := os.Getenv("MAINTENANCE_MODE")
- if msg != "" {
- return nil, errors.HTTP(http.StatusTeapot, errors.Errorf("maintenance mode"), "maintenance mode")
- }
- return nil, nil //not maint mode
-}
-
-type rateLimiter struct{}
-
-func (r rateLimiter) Check(ctx api.Context) (interface{}, error) {
- if rand.Intn(10) < 2 {
- return nil, errors.Errorf("rate limited")
- }
- return nil, nil //not limited
-}
-
-type cors struct{}
-
-func (cors) CORS() map[string]string {
- return map[string]string{
- "Access-Control-Allow-Origin": "*",
- "Access-Control-Allow-Headers": "Content-Type, Accept",
- }
-}
diff --git a/examples/core/app/app.go b/examples/core/app/app.go
deleted file mode 100644
index a1de7c5..0000000
--- a/examples/core/app/app.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package app
-
-import (
- "gitlab.com/uafrica/go-utils/cron"
- "gitlab.com/uafrica/go-utils/examples/core/app/users"
- "gitlab.com/uafrica/go-utils/examples/core/email"
-)
-
-func ApiRoutes() map[string]map[string]interface{} {
- return map[string]map[string]interface{}{
- "/users": {
- "GET": users.Get,
- "POST": users.Add,
- "PUT": users.Upd,
- "DELETE": users.Del,
- },
- "/crash": {
- "GET": users.Crash,
- },
- }
-}
-
-func QueueRoutes() map[string]interface{} {
- return map[string]interface{}{
- "email": email.Notify,
- }
-}
-
-func CronRoutes() map[string]func(cron.Context) error {
- return map[string]func(cron.Context) error{
- "janitor": users.Janitor,
- }
-}
diff --git a/examples/core/app/users/users.go b/examples/core/app/users/users.go
deleted file mode 100644
index a7dd6cb..0000000
--- a/examples/core/app/users/users.go
+++ /dev/null
@@ -1,156 +0,0 @@
-package users
-
-import (
- "fmt"
- "net/http"
- "sync"
- "time"
-
- "gitlab.com/uafrica/go-utils/api"
- "gitlab.com/uafrica/go-utils/cron"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/examples/core/email"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-type User struct {
- ID int `json:"id"`
- AccountID int `json:"account_id"`
- Username string `json:"username"`
- Password string `json:"password"`
- Email string `json:"email"`
-}
-
-func (u User) Validate() error {
- logger.Debugf("Validating (%T)%+v", u, u)
- if u.ID != 0 {
- return errors.Errorf("id may not be specified for new user")
- }
- if u.AccountID == 0 {
- return errors.Errorf("missing account_id (from claims)")
- }
- if u.Email == "" {
- return errors.Errorf("missing email")
- }
- if u.Username == "" {
- return errors.Errorf("missing username")
- }
- if u.Password == "" {
- return errors.Errorf("missing password")
- }
- return nil
-}
-
-//a simple in memory list of users for this example
-var (
- usersMutex sync.Mutex
- users = map[int]User{} //index on user.ID
- nextUserID = 1
-)
-
-type getParams struct {
- ID int `json:"id"`
-}
-
-func (p getParams) Validate() error {
- if p.ID <= 0 {
- return errors.Errorf("id>0 is required")
- }
- return nil
-}
-
-func Get(ctx api.Context, params getParams) (User, error) {
- db := ctx.Value("db").(int)
- ctx.Debugf("DB = %v", db)
- ctx.Debugf("Claim: %+v", ctx.Claim())
- ctx.Debugf("Params: %+v", params)
- usersMutex.Lock()
- defer usersMutex.Unlock()
- if user, ok := users[params.ID]; ok {
- return user, nil
- }
- return User{}, errors.HTTP(http.StatusNotFound, errors.Errorf("user.id=%d not found", params.ID), "")
-}
-
-type POSTUser struct {
- User
- U1 User `json:"u1"`
- U2 *User `json:"u2"`
-}
-
-func Add(ctx api.Context, params noParams, newUser POSTUser) (User, error) {
- db := ctx.Value("db").(int)
- ctx.Debugf("DB = %v", db)
- ctx.Debugf("Claim: %+v", ctx.Claim())
-
- usersMutex.Lock()
- defer usersMutex.Unlock()
-
- //u1 and u2 not used - just to demonstrate how claims are populated in sub struct and sub struct ptrs
- ctx.Debugf("u1: %+v", newUser.U1)
- if newUser.U2 != nil {
- ctx.Debugf("u2: %+v", newUser.U2)
- }
-
- //make sure user is unique for this account
- for _, u := range users {
- if u.AccountID == newUser.AccountID && u.Username == newUser.Username {
- return User{}, errors.HTTP(http.StatusBadRequest, errors.Errorf("username \"%s\" already exists in this account", newUser.Username), "")
- }
- }
-
- newUser.ID = nextUserID
- nextUserID++
- users[newUser.ID] = newUser.User
-
- //send notification by email
- email := email.Message{
- To: []string{newUser.Email},
- CC: nil,
- BCC: nil,
- From: "example@uafrica.com",
- Subject: "Welcome User",
- Body: "Your account has been created",
- }
- /*eventID*/ _, err := ctx.NewEvent("notify").RequestID(ctx.RequestID()).Type("email").Delay(time.Second * 5).Params(map[string]string{}).Send(email)
- if err != nil {
- ctx.Errorf("failed to notify: %+v", err)
- }
- //ctx.Debugf("Notified: %v", eventID)
- return newUser.User, nil
-}
-
-func Upd(ctx api.Context, params noParams, updUser User) (User, error) {
- usersMutex.Lock()
- defer usersMutex.Unlock()
- existingUser, ok := users[updUser.ID]
- if !ok {
- return User{}, errors.HTTP(http.StatusNotFound, errors.Errorf("user.id=%d not found", updUser.ID), "")
- }
- if existingUser.Username != updUser.Username {
- return User{}, errors.HTTP(http.StatusBadRequest, errors.Errorf("user.id=%d username=%s may not change", existingUser.ID, existingUser.Username), "")
- }
- users[updUser.ID] = updUser
- return updUser, nil
-}
-
-func Del(ctx api.Context, params getParams) error {
- usersMutex.Lock()
- defer usersMutex.Unlock()
- delete(users, params.ID)
- return nil
-}
-
-type noParams struct{}
-
-func Crash(ctx api.Context, params noParams) error {
- var d *int
- fmt.Printf("%d", *d) //this should crash - causing the crash dumper to trigger
- return nil
-}
-
-func Janitor(ctx cron.Context) error {
- db := ctx.Value("db").(int)
- ctx.Debugf("DB = %v", db)
- return errors.Errorf("NYI")
-}
diff --git a/examples/core/cron/main.go b/examples/core/cron/main.go
deleted file mode 100644
index 66a494b..0000000
--- a/examples/core/cron/main.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package main
-
-import (
- "flag"
- "fmt"
- "math/rand"
- "net/http"
- "os"
- "time"
-
- "gitlab.com/uafrica/go-utils/api"
- "gitlab.com/uafrica/go-utils/cron"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/examples/core/app"
- "gitlab.com/uafrica/go-utils/examples/core/db"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-func main() {
- invokeArnPtr := flag.String("arn", "", "Invoke this ARN and terminate (default is to run as lambda)")
- flag.Parse()
-
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.SetGlobalFormat(logger.NewConsole())
-
- cron.New(app.CronRoutes()).
- WithStarter("db", db.Connector("core")).
- Run(invokeArnPtr)
-}
-
-type maint struct{}
-
-//for maintenance mode, put a message in environment variable MAINTENANCE_MODE
-//then than message will be displayed in the response. Clear the variable to
-//proceed normal operation
-func (m maint) Check(ctx api.Context) (interface{}, error) {
- msg := os.Getenv("MAINTENANCE_MODE")
- if msg != "" {
- return nil, errors.HTTP(http.StatusTeapot, errors.Errorf("maintenance mode"), "maintenance mode")
- }
- return nil, nil //not maint mode
-}
-
-type rateLimiter struct{}
-
-func (r rateLimiter) Check(ctx api.Context) (interface{}, error) {
- if rand.Intn(10) < 2 {
- return nil, errors.Errorf("rate limited")
- }
- return nil, nil //not limited
-}
-
-type cors struct{}
-
-func (cors) CORS() map[string]string {
- return map[string]string{
- "Access-Control-Allow-Origin": "*",
- "Access-Control-Allow-Headers": "Content-Type, Accept",
- }
-}
-
-type audit struct{}
-
-func (a audit) Audit(startTime, endTime time.Time, values map[string]interface{}) {
- fmt.Printf("AUDIT: %v %v %v ...\n", startTime, endTime, endTime.Sub(startTime))
-}
diff --git a/examples/core/db/database.go b/examples/core/db/database.go
deleted file mode 100644
index f694c23..0000000
--- a/examples/core/db/database.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package db
-
-import (
- "math/rand"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/service"
-)
-
-func Connector(dbName string) service.Starter {
- return &connector{
- name: dbName,
- conn: 0,
- }
-}
-
-//connector implements service.Starter
-type connector struct {
- name string
- conn int
-}
-
-//Start returns app specific data on success that can be retrieved by handlers
-func (c *connector) Start(ctx service.Context) (interface{}, error) {
- //make reusable db connection
- if c.conn < 2 {
- c.conn = rand.Intn(10)
- if c.conn < 2 {
- return nil, errors.Errorf("failed to connect to db (just a random fake event - try again :-))")
- }
- ctx.Debugf("Connected to db(%s): %d", c.name, c.conn)
- }
-
- //return app data - here we only return db conn,
- //which can be retrieved from ctx.Value("db"),
- //because dbConnector{} is registered as "db"
- return c.conn, nil
-}
diff --git a/examples/core/email/notify.go b/examples/core/email/notify.go
deleted file mode 100644
index 07a1149..0000000
--- a/examples/core/email/notify.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package email
-
-import "gitlab.com/uafrica/go-utils/service"
-
-type Message struct {
- From string
- To []string
- CC []string
- BCC []string
- Subject string
- Body string
-}
-
-func Notify(ctx service.Context, msg Message) error {
- ctx.Debugf("Pretending to send email: %+v", msg)
- return nil
-}
diff --git a/examples/core/sqs/main.go b/examples/core/sqs/main.go
deleted file mode 100644
index daf804c..0000000
--- a/examples/core/sqs/main.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package main
-
-import (
- "gitlab.com/uafrica/go-utils/consumer/sqs_consumer"
- "gitlab.com/uafrica/go-utils/examples/core/db"
-)
-
-func main() {
- sqsRoutes := map[string]interface{}{}
- consumer := sqs_consumer.New("uafrica-request-id", sqsRoutes).
- WithStarter("db", db.Connector("core"))
- consumer.Run()
-}
diff --git a/go.mod b/go.mod
index da1c829..059784e 100644
--- a/go.mod
+++ b/go.mod
@@ -3,35 +3,42 @@ module gitlab.com/uafrica/go-utils
go 1.17
require (
+ github.com/MindscapeHQ/raygun4go v1.1.1
github.com/aws/aws-lambda-go v1.26.0
- github.com/aws/aws-sdk-go v1.40.50
- github.com/cespare/xxhash/v2 v2.1.1 // indirect
+ github.com/aws/aws-sdk-go v1.42.12
+ github.com/aws/aws-secretsmanager-caching-go v1.1.0
+ github.com/go-pg/pg/v10 v10.10.6
+ github.com/go-redis/redis/v8 v8.11.4
+ github.com/go-redis/redis_rate/v9 v9.1.2
+ github.com/google/uuid v1.3.0
+ github.com/opensearch-project/opensearch-go v1.0.0
+ github.com/pkg/errors v0.9.1
+ github.com/r3labs/diff/v2 v2.14.2
+ github.com/sirupsen/logrus v1.8.1
+ github.com/thoas/go-funk v0.9.1
+ golang.org/x/text v0.3.7
+)
+
+require (
+ github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
- github.com/fatih/color v1.13.0
- github.com/go-pg/pg/v10 v10.10.5
+ github.com/go-errors/errors v1.4.1 // indirect
github.com/go-pg/zerochecker v0.2.0 // indirect
- github.com/go-redis/redis/v8 v8.11.3
- github.com/google/uuid v1.3.0
+ github.com/golang/protobuf v1.5.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
- github.com/mattn/go-colorable v0.1.9 // indirect
- github.com/mattn/go-isatty v0.0.14 // indirect
- github.com/pkg/errors v0.9.1
- github.com/thoas/go-funk v0.9.1
+ github.com/pborman/uuid v1.2.1 // indirect
+ github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/vmihailenco/bufpool v0.1.11 // indirect
+ github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
- golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
- golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
- golang.org/x/text v0.3.7
- mellium.im/sasl v0.2.1 // indirect
- github.com/golang/protobuf v1.5.2 // indirect
- github.com/opensearch-project/opensearch-go v1.0.0
- github.com/r3labs/diff/v2 v2.14.0
- github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
+ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+ golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.26.0 // indirect
+ mellium.im/sasl v0.2.1 // indirect
)
diff --git a/go.sum b/go.sum
index f98637f..8e11358 100644
--- a/go.sum
+++ b/go.sum
@@ -1,15 +1,17 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/MindscapeHQ/raygun4go v1.1.1 h1:fk3Uknv9kQxUIwL3mywwHQRyfq3PaR9lE/e40K+OcY0=
+github.com/MindscapeHQ/raygun4go v1.1.1/go.mod h1:NW0eWi2Qs00ZcctO6owrVMY+h2HxzJVgQGDrTj2ysw4=
github.com/aws/aws-lambda-go v1.26.0 h1:6ujqBpYF7tdZcBvPIccs98SpeGfrt/UOVEiexfNIdHA=
github.com/aws/aws-lambda-go v1.26.0/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU=
github.com/aws/aws-sdk-go v1.19.23/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.40.50 h1:QP4NC9EZWBszbNo2UbG6bbObMtN35kCFb4h0r08q884=
-github.com/aws/aws-sdk-go v1.40.50/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-sdk-go v1.42.12 h1:zVrAgi3/HuMPygZknc+f2KAHcn+Zuq767857hnHBMPA=
+github.com/aws/aws-sdk-go v1.42.12/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-secretsmanager-caching-go v1.1.0 h1:vcV94XGJ9KouXKYBTMqgrBw96Tae8JKLmoUZ5SbaXNo=
github.com/aws/aws-secretsmanager-caching-go v1.1.0/go.mod h1:wahQpJP1dZKMqjGFAjGCqilHkTlN0zReGWocPLbXmxg=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -20,17 +22,19 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/go-pg/pg/v10 v10.10.5 h1:RRW8NqxVu4vgzN9k05TT9rM5X+2VQHcIBRLeK9djMBE=
-github.com/go-pg/pg/v10 v10.10.5/go.mod h1:EmoJGYErc+stNN/1Jf+o4csXuprjxcRztBnn6cHe38E=
+github.com/go-errors/errors v1.4.1 h1:IvVlgbzSsaUNudsw5dcXSzF3EWyXTi5XrAdngnuhRyg=
+github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
+github.com/go-pg/pg/v10 v10.10.6 h1:1vNtPZ4Z9dWUw/TjJwOfFUbF5nEq1IkR6yG8Mq/Iwso=
+github.com/go-pg/pg/v10 v10.10.6/go.mod h1:GLmFXufrElQHf5uzM3BQlcfwV3nsgnHue5uzjQ6Nqxg=
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
-github.com/go-redis/redis/v8 v8.11.3 h1:GCjoYp8c+yQTJfc0n69iwSiHjvuAdruxl7elnZCxgt8=
-github.com/go-redis/redis/v8 v8.11.3/go.mod h1:xNJ9xDG09FsIPwh3bWdk+0oDWHbtF9rPN0F/oD9XeKc=
+github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
+github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
+github.com/go-redis/redis_rate/v9 v9.1.2 h1:H0l5VzoAtOE6ydd38j8MCq3ABlGLnvvbA1xDSVVCHgQ=
+github.com/go-redis/redis_rate/v9 v9.1.2/go.mod h1:oam2de2apSgRG8aJzwJddXbNu91Iyz1m8IKJE2vpvlQ=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -56,8 +60,11 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@@ -66,15 +73,12 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -88,20 +92,29 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
-github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU=
-github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/opensearch-project/opensearch-go v1.0.0 h1:8Gh7B7Un5BxuxWAgmzleEF7lpOtC71pCgPp7lKr3ca8=
github.com/opensearch-project/opensearch-go v1.0.0/go.mod h1:FrUl/52DBegRYvK7ISF278AXmjDV647lyTnsLGBR7J4=
+github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
+github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/r3labs/diff/v2 v2.14.0 h1:VRI8lhKFP4miM+RlyKkdoT94u7RlFge2S+WAqDScV2Q=
-github.com/r3labs/diff/v2 v2.14.0/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
+github.com/r3labs/diff/v2 v2.14.2 h1:1HVhQKwg1YnoCWzCYlOWYLG4C3yfTudZo5AcrTSgCTc=
+github.com/r3labs/diff/v2 v2.14.2/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -116,7 +129,6 @@ github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6cz
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
-github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc=
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
@@ -128,8 +140,8 @@ golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b h1:7mWr3k41Qtv8XlltBkDkl8LoP3mpSgBW8BUoxtEdbXg=
-golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -161,18 +173,17 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 h1:c20P3CcPbopVp2f7099WLOqSNKURf30Z0uq66HpijZY=
+golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -184,6 +195,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
diff --git a/handler_utils/api.go b/handler_utils/api.go
new file mode 100644
index 0000000..ce89f0c
--- /dev/null
+++ b/handler_utils/api.go
@@ -0,0 +1,178 @@
+package handler_utils
+
+import (
+ "github.com/aws/aws-lambda-go/events"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logs"
+ "reflect"
+ "strconv"
+ "strings"
+)
+
+// ValidateAPIEndpoints checks that all API endpoints are correctly defined using one of the supported handler types
+// return updated endpoints with additional information
+func ValidateAPIEndpoints(endpoints map[string]map[string]interface{}) (map[string]map[string]interface{}, error) {
+ countLegacy := 0
+ countHandler := 0
+ for resource, methodHandlers := range endpoints {
+ if resource == "" {
+ return nil, errors.Errorf("blank resource")
+ }
+ if resource == "/api-docs" {
+ return nil, errors.Errorf("%s may not be a defined endpoint - it is reserved", resource)
+ }
+ for method, handlerFunc := range methodHandlers {
+ switch method {
+ case "GET":
+ case "POST":
+ case "PUT":
+ case "PATCH":
+ case "DELETE":
+ default:
+ return nil, errors.Errorf("Invalid method:\"%s\" on resource \"%s\"", method, resource)
+ }
+ if handlerFunc == nil {
+ return nil, errors.Errorf("nil handler on %s %s", method, resource)
+ }
+
+ if _, ok := handlerFunc.(func(req events.APIGatewayProxyRequest) (response events.APIGatewayProxyResponse, err error)); ok {
+ //ok - leave as is - we support this legacyHandler
+ countLegacy++
+ } else {
+ handler, err := NewHandler(handlerFunc)
+ if err != nil {
+ return nil, errors.Wrapf(err, "%s %s has invalid handler %T", method, resource, handlerFunc)
+ }
+ //replace the endpoint value so that we can quickly call this handler
+ endpoints[resource][method] = handler
+ countHandler++
+ }
+ }
+ }
+ logs.Info("Checked %d legacy and %d new handlers\n", countLegacy, countHandler)
+ return endpoints, nil
+}
+
+func ValidateRequestParams(request *events.APIGatewayProxyRequest, paramsStructType reflect.Type) (reflect.Value, error) {
+ paramValues := map[string]interface{}{}
+ for n, v := range request.QueryStringParameters {
+ paramValues[n] = v
+ }
+ paramsStructValuePtr := reflect.New(paramsStructType)
+ for i := 0; i < paramsStructType.NumField(); i++ {
+ f := paramsStructType.Field(i)
+ n := (strings.SplitN(f.Tag.Get("json"), ",", 2))[0]
+ if n == "" {
+ n = strings.ToLower(f.Name)
+ }
+ if n == "" || n == "-" {
+ continue
+ }
+
+ //get value(s) from query string
+ var paramStrValues []string
+ if paramStrValue, isDefined := request.QueryStringParameters[n]; isDefined {
+ paramStrValues = []string{paramStrValue} //single value
+ } else {
+ paramStrValues = request.MultiValueQueryStringParameters[n]
+ }
+ if len(paramStrValues) == 0 {
+ continue //param has no value specified in URL
+ }
+
+ //param is defined >=1 times in URL
+ if f.Type.Kind() == reflect.Slice {
+ //iterate over all specified values
+ for _, paramStrValue := range paramStrValues {
+ newValuePtr := reflect.New(f.Type.Elem())
+ if err := setParamFromStr(
+ newValuePtr.Elem(),
+ paramStrValue); err != nil {
+ return reflect.Value{}, errors.Wrapf(err, "failed to set %s[%d]=%s", n, i, paramStrValues[0])
+ }
+ paramsStructValuePtr.Elem().Field(i).Set(reflect.Append(paramsStructValuePtr.Elem().Field(i), newValuePtr.Elem()))
+ }
+ } else {
+ if len(paramStrValues) > 1 {
+ return reflect.Value{}, errors.Errorf("%s does not support >1 values(%v)", n, strings.Join(paramStrValues, ","))
+ }
+ //single value specified
+ if err := setParamFromStr(paramsStructValuePtr.Elem().Field(i), paramStrValues[0]); err != nil {
+ return reflect.Value{}, errors.Wrapf(err, "failed to set %s=%s", n, paramStrValues[0])
+ }
+ }
+ } //for each param struct field
+
+ return paramsStructValuePtr, nil
+}
+
+
+
+func setParamFromStr(fieldValue reflect.Value, paramStrValue string) error {
+ switch fieldValue.Type().Kind() {
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+ //parse to int for this struct field
+ if i64, err := strconv.ParseInt(paramStrValue, 10, 64); err != nil {
+ return errors.Errorf("%s is not a number", paramStrValue)
+ } else {
+ switch fieldValue.Type().Kind() {
+ case reflect.Int:
+ fieldValue.Set(reflect.ValueOf(int(i64)))
+ case reflect.Int8:
+ fieldValue.Set(reflect.ValueOf(int8(i64)))
+ case reflect.Int16:
+ fieldValue.Set(reflect.ValueOf(int16(i64)))
+ case reflect.Int32:
+ fieldValue.Set(reflect.ValueOf(int32(i64)))
+ case reflect.Int64:
+ fieldValue.Set(reflect.ValueOf(i64))
+ }
+ }
+
+ case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+ //parse to int for this struct field
+ if u64, err := strconv.ParseUint(paramStrValue, 10, 64); err != nil {
+ return errors.Errorf("%s is not a number", paramStrValue)
+ } else {
+ switch fieldValue.Type().Kind() {
+ case reflect.Uint:
+ fieldValue.Set(reflect.ValueOf(uint(u64)))
+ case reflect.Uint8:
+ fieldValue.Set(reflect.ValueOf(uint8(u64)))
+ case reflect.Uint16:
+ fieldValue.Set(reflect.ValueOf(uint16(u64)))
+ case reflect.Uint32:
+ fieldValue.Set(reflect.ValueOf(uint32(u64)))
+ case reflect.Uint64:
+ fieldValue.Set(reflect.ValueOf(u64))
+ }
+ }
+
+ case reflect.Bool:
+ bs := strings.ToLower(paramStrValue)
+ if bs == "true" || bs == "yes" || bs == "1" {
+ fieldValue.Set(reflect.ValueOf(true))
+ }
+
+ case reflect.String:
+ fieldValue.Set(reflect.ValueOf(paramStrValue))
+
+ case reflect.Float32:
+ if f32, err := strconv.ParseFloat(paramStrValue, 32); err != nil {
+ return errors.Wrapf(err, "invalid float")
+ } else {
+ fieldValue.Set(reflect.ValueOf(float32(f32)))
+ }
+
+ case reflect.Float64:
+ if f64, err := strconv.ParseFloat(paramStrValue, 64); err != nil {
+ return errors.Wrapf(err, "invalid float")
+ } else {
+ fieldValue.Set(reflect.ValueOf(f64))
+ }
+
+ default:
+ return errors.Errorf("unsupported type %v", fieldValue.Type().Kind())
+ } //switch param struct field
+ return nil
+}
diff --git a/handler_utils/cron.go b/handler_utils/cron.go
new file mode 100644
index 0000000..8fbd9b3
--- /dev/null
+++ b/handler_utils/cron.go
@@ -0,0 +1,23 @@
+package handler_utils
+
+import (
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logs"
+)
+
+// ValidateCronHandlers checks that all handlers are correctly defined using one of the supported handler types
+// return updated handlers (with additional information -> N/A for cron)
+func ValidateCronHandlers(handlers map[string]func() error) (map[string]func() error, error) {
+ countHandler := 0
+ for cronName, cronFunc := range handlers {
+ if cronName == "" {
+ return nil, errors.Errorf("blank handlerName")
+ }
+ if cronFunc == nil {
+ return nil, errors.Errorf("nil handler on %s", cronName)
+ }
+ countHandler++
+ }
+ logs.Info("Checked %d handlers\n", countHandler)
+ return handlers, nil
+}
diff --git a/handler_utils/handler.go b/handler_utils/handler.go
new file mode 100644
index 0000000..89c8c8a
--- /dev/null
+++ b/handler_utils/handler.go
@@ -0,0 +1,114 @@
+package handler_utils
+
+import (
+ "reflect"
+
+ "gitlab.com/uafrica/go-utils/errors"
+)
+
+type Handler struct {
+ RequestParamsType reflect.Type
+ RequestBodyType reflect.Type
+ ResponseType reflect.Type
+ FuncValue reflect.Value
+}
+
+type SQSHandler struct {
+ RecordType reflect.Type
+ FuncValue reflect.Value
+}
+
+func NewHandler(handlerFunction interface{}) (Handler, error) {
+ h := Handler{}
+
+ handlerFunctionType := reflect.TypeOf(handlerFunction)
+ if handlerFunctionType.NumIn() < 1 || handlerFunctionType.NumIn() > 2 {
+ return h, errors.Errorf("takes %d args instead of (Params[, Body])", handlerFunctionType.NumIn())
+ }
+ if handlerFunctionType.NumOut() < 1 || handlerFunctionType.NumOut() > 2 {
+ return h, errors.Errorf("returns %d results instead of ([Response,] error)", handlerFunctionType.NumOut())
+ }
+
+ //arg[0] must be a struct for params. It may be an empty struct, but
+ //all public fields require a json tag which we will use to math the URL param name
+ if err := validateStructType(handlerFunctionType.In(0)); err != nil {
+ return h, errors.Wrapf(err, "first arg %v is not valid params struct type", handlerFunctionType.In(0))
+ }
+ h.RequestParamsType = handlerFunctionType.In(0)
+
+ //arg[1] is optional and must be a struct for request body. It may be an empty struct, but
+ //all public fields require a json tag which we will use to unmarshal the request body from JSON
+ if handlerFunctionType.NumIn() >= 2 {
+ if handlerFunctionType.In(1).Kind() == reflect.Slice {
+ if err := validateStructType(handlerFunctionType.In(1).Elem()); err != nil {
+ return h, errors.Errorf("second arg %v is not valid body []struct type", handlerFunctionType.In(1))
+ }
+ } else {
+ if err := validateStructType(handlerFunctionType.In(1)); err != nil {
+ return h, errors.Errorf("second arg %v is not valid body struct type", handlerFunctionType.In(1))
+ }
+ }
+
+ //todo: check special fields for claims, and see if also applies to params struct...
+ //AccountID must be int64 or *int64 with tag =???
+ //UserID must be int64 or *int64 with tag =???
+ //Username must be string with tag =???
+
+ h.RequestBodyType = handlerFunctionType.In(1)
+ }
+
+ //last result must be error
+ if _, ok := reflect.New(handlerFunctionType.Out(handlerFunctionType.NumOut() - 1)).Interface().(*error); !ok {
+ return h, errors.Errorf("last result %v is not error type", handlerFunctionType.Out(handlerFunctionType.NumOut()-1))
+ }
+
+ h.FuncValue = reflect.ValueOf(handlerFunction)
+ return h, nil
+}
+
+func validateStructType(t reflect.Type) error {
+ if t.Kind() != reflect.Struct {
+ return errors.Errorf("%v is %v, not a struct", t, t.Kind())
+ }
+ for i := 0; i < t.NumField(); i++ {
+ f := t.Field(i)
+ if f.Name[0] >= 'a' && f.Name[0] <= 'z' {
+ //lowercase fields should not have json tag
+ if f.Tag.Get("json") != "" {
+ return errors.Errorf("%s.%s must be uppercase because it has a json tag \"%s\"",
+ t.Name(),
+ f.Name,
+ f.Tag.Get("json"))
+ }
+ }
+
+ // if f.... check tags recursively... for now, not too strict ... add checks if we see issues that break the API, to help dev to fix before we deploy, or to prevent bad habits...
+ }
+ return nil
+}
+
+func NewSQSHandler(fnc interface{}) (SQSHandler, error) {
+ h := SQSHandler{}
+
+ fncType := reflect.TypeOf(fnc)
+ if fncType.NumIn() != 1 {
+ return h, errors.Errorf("takes %d args instead of (Record)", fncType.NumIn())
+ }
+ if fncType.NumOut() != 1 {
+ return h, errors.Errorf("returns %d results instead of (error)", fncType.NumOut())
+ }
+
+ // Arg[0] must be a struct for the message record body.
+ if fncType.In(0).Kind() != reflect.Struct {
+ return h, errors.Errorf("first arg %v is not valid record struct type", fncType.In(0))
+ }
+ h.RecordType = fncType.In(0)
+
+ // Result must be error
+ if _, ok := reflect.New(fncType.Out(0)).Interface().(*error); !ok {
+ return h, errors.Errorf("result %v is not error type", fncType.Out(0))
+ }
+
+ h.FuncValue = reflect.ValueOf(fnc)
+ return h, nil
+}
diff --git a/handler_utils/request.go b/handler_utils/request.go
new file mode 100644
index 0000000..f3e4260
--- /dev/null
+++ b/handler_utils/request.go
@@ -0,0 +1,32 @@
+package handler_utils
+
+import (
+ "context"
+
+ "github.com/aws/aws-lambda-go/lambdacontext"
+)
+
+func RequestIDFromLambdaContext(ctx context.Context) *string {
+ // Get request ID from context
+ if ctx != nil {
+ lambdaContext, _ := lambdacontext.FromContext(ctx)
+ if lambdaContext != nil {
+ return &lambdaContext.AwsRequestID
+ }
+ }
+ return nil
+}
+
+func RequestIDFromHeaders(headers map[string]string, requestIDHeaderKey string) *string {
+ // Get request ID from parent micro service
+ if requestID, ok := headers[requestIDHeaderKey]; ok {
+ return &requestID
+ }
+ return nil
+}
+
+func AddRequestIDToHeaders(requestID *string, headers map[string]string, requestIDHeaderKey string) {
+ if requestID != nil && headers != nil {
+ headers[requestIDHeaderKey] = *requestID
+ }
+}
diff --git a/handler_utils/sqs.go b/handler_utils/sqs.go
new file mode 100644
index 0000000..837ad9c
--- /dev/null
+++ b/handler_utils/sqs.go
@@ -0,0 +1,60 @@
+package handler_utils
+
+import (
+ "encoding/json"
+ "github.com/aws/aws-lambda-go/events"
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logs"
+ "reflect"
+)
+
+// ValidateSQSEndpoints checks that all SQS endpoints are correctly defined using one of the supported handler types
+// return updated endpoints with additional information
+func ValidateSQSEndpoints(endpoints map[string]interface{}) (map[string]interface{}, error) {
+ countLegacy := 0
+ countHandler := 0
+ for messageType, handlerFunc := range endpoints {
+ if messageType == "" {
+ return nil, errors.Errorf("blank messageType")
+ }
+ if handlerFunc == nil {
+ return nil, errors.Errorf("nil handler on %s", messageType)
+ }
+
+ if _, ok := handlerFunc.(func(req events.SQSEvent) (err error)); ok {
+ // ok - leave as is - we support this legacyHandler
+ countLegacy++
+ } else {
+ handler, err := NewSQSHandler(handlerFunc)
+ if err != nil {
+ return nil, errors.Wrapf(err, "%v has invalid handler %T", messageType, handlerFunc)
+ }
+ // replace the endpoint value so we can quickly call this handler
+ endpoints[messageType] = handler
+ logs.Info("%s: OK (request: %v)\n", messageType, handler.RecordType)
+ countHandler++
+ }
+ }
+ logs.Info("Checked %d legacy and %d new handlers\n", countLegacy, countHandler)
+ return endpoints, nil
+}
+
+func GetRecord(message events.SQSMessage, recordType reflect.Type) (interface{}, error) {
+ recordValuePtr := reflect.New(recordType)
+ err := json.Unmarshal([]byte(message.Body), recordValuePtr.Interface())
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to JSON decode message body")
+ }
+
+ if validator, ok := recordValuePtr.Interface().(IValidator); ok {
+ if err := validator.Validate(); err != nil {
+ return nil, errors.Wrapf(err, "invalid message body")
+ }
+ }
+
+ return recordValuePtr.Elem().Interface(), nil
+}
+
+type IValidator interface {
+ Validate() error
+}
diff --git a/logger/context.go b/logger/context.go
deleted file mode 100644
index 244ba31..0000000
--- a/logger/context.go
+++ /dev/null
@@ -1,44 +0,0 @@
-package logger
-
-import (
- "fmt"
-)
-
-func GetContextLogger() Logger {
- return globalLogger
-}
-
-func LogMessageWithFields(fields map[string]interface{}, message interface{}) {
- globalLogger.withFields(fields).log(LevelInfo, 1, fmt.Sprintf("%v", message))
-}
-
-func LogMessage(format string, a ...interface{}) {
- globalLogger.log(LevelInfo, 1, fmt.Sprintf(format, a...))
-}
-
-func LogError(fields map[string]interface{}, err error) {
- // sendRaygunError(fields, err)
- globalLogger.withFields(fields).log(LevelError, 1, fmt.Sprintf("%+v", err))
-}
-
-func LogErrorMessage(message interface{}, err error) {
- if err != nil || message != nil {
- globalLogger.withFields(map[string]interface{}{
- "error": fmt.Sprintf("%+v", err),
- }).log(LevelError, 1, fmt.Sprintf("%v", message))
- }
-}
-
-func LogWarningMessage(format string, a ...interface{}) {
- globalLogger.log(LevelWarn, 1, fmt.Sprintf(format, a...))
-}
-
-func LogWarning(fields map[string]interface{}, err error) {
- globalLogger.withFields(fields).log(LevelWarn, 1, fmt.Sprintf("%+v", err))
-}
-
-func SQLDebugInfo(sql string) {
- globalLogger.withFields(map[string]interface{}{
- "sql": sql,
- }).log(LevelInfo, 1, "SQL")
-}
diff --git a/logger/db.go b/logger/db.go
deleted file mode 100644
index 5186700..0000000
--- a/logger/db.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package logger
-
-import (
- "context"
- "os"
-
- "github.com/go-pg/pg/v10"
-)
-
-type DbLogger struct{}
-
-func (d DbLogger) BeforeQuery(c context.Context, _ *pg.QueryEvent) (context.Context, error) {
- return c, nil
-}
-
-func (d DbLogger) AfterQuery(_ context.Context, q *pg.QueryEvent) error {
- sql, _ := q.FormattedQuery()
- if os.Getenv("DEBUGGING") == "true" {
- Debugf(string(sql))
- } else {
- SQLDebugInfo(string(sql))
- }
- return nil
-}
diff --git a/logger/format.go b/logger/format.go
deleted file mode 100644
index 3ee4993..0000000
--- a/logger/format.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package logger
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
-
- "github.com/fatih/color"
-)
-
-var nextFormatID = 1
-
-type IFormatter interface {
- Format(Entry) []byte
- NextColor() IFormatter
- Color() string
-}
-
-type formatterJSON struct{}
-
-func (f formatterJSON) Format(entry Entry) []byte {
- jsonEntry, err := json.Marshal(entry)
- if err != nil {
- return []byte(fmt.Sprintf("failed to marshal entry: %v: %+v\n", err, entry))
- }
- return append(jsonEntry, []byte("\n")...)
-}
-
-func (f formatterJSON) NextColor() IFormatter {
- return f //do not select colors for JSON (only used in console)
-}
-
-func (f formatterJSON) Color() string {
- return "default"
-}
-
-func NewConsole() IFormatter {
- nextFormatID++
- return formatterConsole{
- id: nextFormatID,
- fg: 1, //color.FgWhite,
- bg: 0, //color.BgBlack,
- }
-}
-
-type formatterConsole struct {
- id int
- fg int //color.Attribute
- bg int //color.Attribute
-}
-
-func (f formatterConsole) Format(entry Entry) []byte {
- source := fmt.Sprintf("%s/%s:%d", entry.Caller.Package, entry.Caller.File, entry.Caller.Line)
- if len(source) > 40 {
- source = source[len(source)-40:]
- }
-
- buffer := bytes.NewBuffer(nil)
-
- red := color.New(color.FgRed).FprintfFunc()
- magenta := color.New(color.FgMagenta).FprintfFunc()
- yellow := color.New(color.FgYellow).FprintfFunc()
- green := color.New(color.FgGreen).FprintfFunc()
- cyan := color.New(color.FgCyan).FprintfFunc()
-
- cyan(buffer, entry.Timestamp.Format("2006-01-02 15:04:05"))
-
- levelString := fmt.Sprintf(" %5.5s", entry.Level)
- switch entry.Level {
- case LevelFatal:
- red(buffer, levelString)
- case LevelError:
- red(buffer, levelString)
- case LevelWarn:
- magenta(buffer, levelString)
- case LevelInfo:
- yellow(buffer, levelString)
- case LevelDebug:
- green(buffer, levelString)
- }
- cyan(buffer, fmt.Sprintf(" %-40.40s| ", source))
-
- base := color.New(fgColors[colorNames[f.fg]], bgColors[colorNames[f.bg]]).FprintfFunc()
- base(buffer, entry.Message)
-
- if len(entry.Data) > 0 {
- jsonData, _ := json.Marshal(entry.Data)
- green(buffer, " "+string(jsonData))
- }
- buffer.WriteString("\n")
- return buffer.Bytes()
-}
-
-func (f formatterConsole) WithForeground(fg color.Attribute) IFormatter {
- // f.fg = fg
- return f
-}
-
-func (f formatterConsole) WithBackground(bg color.Attribute) IFormatter {
- // f.bg = bg
- return f
-}
-
-func (f formatterConsole) Color() string {
- return colorNames[f.fg] + " on " + colorNames[f.bg]
-}
-
-var (
- colorNames = []string{"black", "white", "red", "green", "yellow", "blue", "magenta", "cyan"}
- fgColors = map[string]color.Attribute{
- "black": color.FgBlack,
- "white": color.FgWhite,
- "red": color.FgRed,
- "green": color.FgGreen,
- "yellow": color.FgYellow,
- "blue": color.FgBlue,
- "magenta": color.FgMagenta,
- "cyan": color.FgCyan,
- }
- bgColors = map[string]color.Attribute{
- "black": color.BgBlack,
- "white": color.BgWhite,
- "red": color.BgRed,
- "green": color.BgGreen,
- "yellow": color.BgYellow,
- "blue": color.BgBlue,
- "magenta": color.BgMagenta,
- "cyan": color.BgCyan,
- }
- nextFg = 1
- nextBg = 0
-)
-
-func (f formatterConsole) NextColor() IFormatter {
- for {
- nextFg++
- if nextFg >= len(fgColors) {
- nextFg = 0
- }
- if nextFg != nextBg {
- break
- }
- }
- nextFormatID++
- f = formatterConsole{
- id: nextFormatID,
- fg: nextFg,
- bg: nextBg,
- }
- return f
-}
diff --git a/logger/global.go b/logger/global.go
deleted file mode 100644
index c5f2195..0000000
--- a/logger/global.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package logger
-
-import (
- "fmt"
- "os"
-
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-var globalLogger logger
-
-func init() {
- globalLogger = logger{
- level: LevelInfo, //default for production
- writer: os.Stderr,
- data: map[string]interface{}{},
- IFormatter: formatterJSON{}, //default to JSON format for production
- }
-}
-
-func SetGlobalLevel(level Level) {
- globalLogger.level = level
-}
-
-func SetGlobalFormat(f IFormatter) {
- if f != nil {
- globalLogger.IFormatter = f
- }
-}
-
-func New() Logger {
- return globalLogger.WithFields(nil)
-}
-
-//shortcut functions to use current logger
-//this should only be used outside of a request context
-//or anywhere if you have a single threaded process
-func Fatalf(format string, args ...interface{}) {
- globalLogger.withFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprintf(format, args...))
- os.Exit(1)
-}
-
-func Fatal(args ...interface{}) {
- globalLogger.withFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprint(args...))
- os.Exit(1)
-}
-
-func Errorf(format string, args ...interface{}) {
- globalLogger.log(LevelError, 1, fmt.Sprintf(format, args...))
-}
-
-func Error(args ...interface{}) {
- globalLogger.log(LevelError, 1, fmt.Sprint(args...))
-}
-
-func Warnf(format string, args ...interface{}) {
- globalLogger.log(LevelWarn, 1, fmt.Sprintf(format, args...))
-}
-
-func Warn(args ...interface{}) {
- globalLogger.log(LevelWarn, 1, fmt.Sprint(args...))
-}
-
-func Infof(format string, args ...interface{}) {
- globalLogger.log(LevelInfo, 1, fmt.Sprintf(format, args...))
-}
-
-func Info(args ...interface{}) {
- globalLogger.log(LevelInfo, 1, fmt.Sprint(args...))
-}
-
-func Debugf(format string, args ...interface{}) {
- globalLogger.log(LevelDebug, 1, fmt.Sprintf(format, args...))
-}
-
-func Debug(args ...interface{}) {
- globalLogger.log(LevelDebug, 1, fmt.Sprint(args...))
-}
-
-func Tracef(format string, args ...interface{}) {
- globalLogger.log(LevelTrace, 1, fmt.Sprintf(format, args...))
-}
-
-func Trace(args ...interface{}) {
- globalLogger.log(LevelTrace, 1, fmt.Sprint(args...))
-}
diff --git a/logger/level.go b/logger/level.go
deleted file mode 100644
index 2546da6..0000000
--- a/logger/level.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package logger
-
-import "fmt"
-
-type Level int
-
-func (level Level) String() string {
- switch level {
- case LevelFatal:
- return "fatal"
- case LevelError:
- return "error"
- case LevelWarn:
- return "warn"
- case LevelInfo:
- return "info"
- case LevelDebug:
- return "debug"
- case LevelTrace:
- return "trace"
- }
- return fmt.Sprintf("Level(%d)", level)
-}
-
-func (level Level) MarshalJSON() ([]byte, error) {
- return []byte("\"" + level.String() + "\""), nil
-}
-
-const (
- LevelFatal Level = iota
- LevelError
- LevelWarn
- LevelInfo
- LevelDebug
- LevelTrace
-)
diff --git a/logger/logger.go b/logger/logger.go
deleted file mode 100644
index 2f65ed0..0000000
--- a/logger/logger.go
+++ /dev/null
@@ -1,238 +0,0 @@
-package logger
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "regexp"
- "strings"
- "time"
-
- "gitlab.com/uafrica/go-utils/errors"
-)
-
-type Logger interface {
- Fatalf(format string, args ...interface{})
- Fatal(args ...interface{})
- Errorf(format string, args ...interface{})
- Error(args ...interface{})
- Warnf(format string, args ...interface{})
- Warn(args ...interface{})
- Infof(format string, args ...interface{})
- Info(args ...interface{})
- Debugf(format string, args ...interface{})
- Debug(args ...interface{})
- Tracef(format string, args ...interface{})
- Trace(args ...interface{})
-
- WithFields(data map[string]interface{}) Logger
- WithSensitiveWord(word string) Logger
- SensitiveWords() []string
-}
-
-type logger struct {
- level Level
- writer io.Writer
- data map[string]interface{}
- sensitiveWords map[string]bool //map key is lowercase word
- IFormatter
-}
-
-func (l logger) WithFields(data map[string]interface{}) Logger {
- l = l.withFields(data)
- return l
-}
-
-func (l logger) withFields(data map[string]interface{}) logger {
- newLogger := logger{
- level: l.level,
- writer: l.writer,
- data: map[string]interface{}{},
- IFormatter: l.IFormatter,
- }
- for n, v := range l.data {
- newLogger.data[n] = v
- }
- for n, v := range data {
- newLogger.data[n] = v
- }
- return newLogger
-}
-
-//word may only consist of alpha-numerics with delimeters inside,
-//e.g. OTP "12345" or Card number "1234-12345678-12345678-1234"
-//and the delimiters may be spaces, dashes, underscores, slashes and dots
-//and it is considered case insensitive
-const sensitiveWordPattern = `[a-z0-9]([a-z0-9\.\\\/ _-]*[a-z0-9])*`
-
-var sensitiveWordRegex = regexp.MustCompile("^" + sensitiveWordPattern + "$")
-
-func (l logger) WithSensitiveWord(word string) Logger {
- l = l.withSensitiveWord(word)
- return l
-}
-
-func (l logger) withSensitiveWord(word string) logger {
- word = strings.ToLower(word)
- if !sensitiveWordRegex.MatchString(word) {
- l.Errorf("cannot add \"%s\" as sensitive word, expecting %s", word, sensitiveWordPattern)
- return l
- }
- if l.sensitiveWords == nil {
- l.sensitiveWords = map[string]bool{word: true}
- } else {
- l.sensitiveWords[word] = true
- }
- return l
-}
-
-func (l logger) SensitiveWords() []string {
- words := make([]string, len(l.sensitiveWords))
- i := 0
- for w := range l.sensitiveWords {
- words[i] = w
- i++
- }
- return words
-}
-
-const delimiters = "()[]{}!@#$%^&*-=_+;:'\"|\\/?<>,.~` \n\r"
-
-func FilterSensitiveWordsMap(s string, wordsMap map[string]bool) (filtered string, changed bool) {
- if len(wordsMap) == 0 {
- return s, false
- }
-
- changed = false
- f := []byte(s)
- for word := range wordsMap {
- //it will be inefficient to compile regex for each word in each context
- //much quicker to just look for the word and see if it is delimited as required
- //not to mach short words as part of longer words which may expose the word be assumption
- //e.g. OTP "202" should not match part of a date 2021-01-02 making it ***1-01-02
- wLen := len(word)
- offset := 0
- fLen := len(f)
- for offset < fLen {
- index := bytes.Index(f[offset:], []byte(word)) + offset
- if index < offset {
- break //word not found
- }
-
- //found the word, check delimiters before/after
- if index > 0 && strings.IndexByte(delimiters, f[index-1]) < 0 {
- offset = index + 1 //word match without delimiter before
- continue
- }
-
- if index+wLen < fLen && strings.IndexByte(delimiters, f[index+wLen]) < 0 {
- offset = index + 1 //word match without delimiter after
- continue
- }
-
- //has delimiter after, this is a word match, replace any length match with 3 stars "***"
- //pad length if required
- pad := 0
- for fLen < index+3 {
- f = append(f, ' ')
- fLen++
- pad++
- }
- f = append(f[:index+3], f[index+wLen:fLen-pad]...)
- f[index] = '*'
- f[index+1] = '*'
- f[index+2] = '*'
- fLen = len(f)
- changed = true
- offset = index + 3 //for loop skipped index over word, now skip offset over delimiter
- }
- }
- filtered = string(f)
- return
-}
-
-func (l logger) Fatalf(format string, args ...interface{}) {
- l.withFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Fatal(args ...interface{}) {
- l.withFields(map[string]interface{}{"call_stack": errors.Stack(3)}).log(LevelFatal, 1, fmt.Sprint(args...))
-}
-
-func (l logger) Errorf(format string, args ...interface{}) {
- l.log(LevelError, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Error(args ...interface{}) {
- l.log(LevelError, 1, fmt.Sprint(args...))
-}
-
-func (l logger) Warnf(format string, args ...interface{}) {
- l.log(LevelWarn, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Warn(args ...interface{}) {
- l.log(LevelWarn, 1, fmt.Sprint(args...))
-}
-
-func (l logger) Infof(format string, args ...interface{}) {
- l.log(LevelInfo, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Info(args ...interface{}) {
- l.log(LevelInfo, 1, fmt.Sprint(args...))
-}
-
-func (l logger) Debugf(format string, args ...interface{}) {
- l.log(LevelDebug, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Debug(args ...interface{}) {
- l.log(LevelDebug, 1, fmt.Sprint(args...))
-}
-
-func (l logger) Tracef(format string, args ...interface{}) {
- l.log(LevelTrace, 1, fmt.Sprintf(format, args...))
-}
-
-func (l logger) Trace(args ...interface{}) {
- l.log(LevelTrace, 1, fmt.Sprint(args...))
-}
-
-func (l logger) log(level Level, skip int, msg string) {
- if level <= l.level && l.writer != nil {
- entry := Entry{
- Timestamp: time.Now(),
- Level: level,
- Caller: errors.GetCaller(skip + 2).Info(),
- Data: l.data,
- Message: strings.ReplaceAll(msg, "\n", ";"),
- }
-
- //filter sensitive words out of data values and message text
- for dataName, dataValue := range entry.Data {
- dataString, ok := dataValue.(string)
- if !ok {
- jsonValue, err := json.Marshal(dataValue)
- if err != nil {
- continue
- }
- dataString = string(jsonValue)
- }
- if filteredString, changed := FilterSensitiveWordsMap(dataString, l.sensitiveWords); changed {
- entry.Data[dataName] = filteredString
- }
- }
- entry.Message, _ = FilterSensitiveWordsMap(entry.Message, l.sensitiveWords)
- l.writer.Write(l.Format(entry))
- }
-}
-
-type Entry struct {
- Timestamp time.Time `json:"time"`
- Level Level `json:"level"`
- Caller errors.CallerInfo `json:"caller"`
- Data map[string]interface{} `json:"data"`
- Message string `json:"message"`
-}
diff --git a/logger/logs_test.go b/logger/logs_test.go
deleted file mode 100644
index 768865d..0000000
--- a/logger/logs_test.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package logger_test
-
-import (
- "os"
- "testing"
-
- "github.com/fatih/color"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-func TestLogs(t *testing.T) {
- //requestID := t.Name()
- //event := events.APIGatewayProxyRequest{}
- os.Setenv("DEBUGGING", "true")
- //logger.InitLogs(&requestID, &event)
-
- // formatter := log.TextFormatter{}
- // log.SetFormatter(&formatter)
-
- logger.LogMessageWithFields(map[string]interface{}{"a": 1, "b": 2}, "MyLogMessage1")
- logger.LogMessage("MyLogMessage2=%d,%d,%d", 1, 2, 3)
- logger.LogError(map[string]interface{}{"a": 4, "b": 5}, errors.Errorf("simple mistake"))
- logger.LogErrorMessage("Error Message", errors.Errorf("another simple mistake"))
- logger.LogWarningMessage("Warning about a=%s,%s,%s", "a", "b", "c")
- logger.LogWarning(map[string]interface{}{"a": 4, "b": 5}, errors.Errorf("Cant believe it failed"))
- logger.SQLDebugInfo("SELECT * from user")
- //logger.LogRequestInfo(event)
- //logs.LogSQSEvent(sqsEvent)
-
- ctx := logger.GetContextLogger()
- ctx.Debugf("Debugging %d!", 456)
- ctx.Infof("Info %d", 789)
-
- //logs.Errorf("Debugging %d!", 456)
- //logs.Error("Info")
-}
-
-func TestColor(t *testing.T) {
- blue := color.New(color.FgBlue).FprintfFunc()
- blue(os.Stdout, "important notice: %s", "ssss")
-
- // Mix up with multiple attributes
- success := color.New(color.Bold, color.FgGreen).FprintlnFunc()
- success(os.Stdout, " don't forget this...")
-
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
- logger.Debugf("Main logger")
- l := logger.New()
- l.Debugf("Logger 1")
- l = logger.New()
- l.Debugf("Logger 2")
- // l = logger.New()
- // l.IFormatter = l.IFormatter.NextColor()
- // l.Debugf("Logger 3")
-}
diff --git a/logs/logs.go b/logs/logs.go
new file mode 100644
index 0000000..75f7d79
--- /dev/null
+++ b/logs/logs.go
@@ -0,0 +1,243 @@
+package logs
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "runtime"
+ "strings"
+
+ "github.com/MindscapeHQ/raygun4go"
+
+ "github.com/aws/aws-lambda-go/events"
+ log "github.com/sirupsen/logrus"
+)
+
+var logger *log.Entry
+
+var apiRequest *events.APIGatewayProxyRequest
+var currentRequestID *string
+var isDebug = false
+var build string
+var raygunClient *raygun4go.Client
+
+// TODO
+// Sensitive word filtering
+
+func InitLogs(requestID *string, isDebugBuild bool, buildVersion string, request *events.APIGatewayProxyRequest, client *raygun4go.Client) {
+ currentRequestID = requestID
+ apiRequest = request
+ isDebug = isDebugBuild
+ build = buildVersion
+ raygunClient = client
+
+ if isDebugBuild {
+ log.SetReportCaller(true)
+ log.SetFormatter(&log.TextFormatter{
+ ForceColors: true,
+ PadLevelText: true,
+ DisableTimestamp: true,
+ CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+ // Exclude the caller, will rather be added as a field
+ return "", ""
+ },
+ })
+ } else {
+ log.SetReportCaller(true)
+ log.SetFormatter(&log.JSONFormatter{
+ CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+ // Exclude the caller, will rather be added as a field
+ return "", ""
+ }})
+ }
+ log.SetLevel(LogLevel())
+
+ val, exists := os.LookupEnv("DEBUGGING")
+ if exists && val == "true" {
+ log.SetLevel(log.TraceLevel)
+ log.SetReportCaller(true)
+ }
+
+ logger = log.WithFields(log.Fields{
+ "environment": getEnvironment(),
+ })
+
+ if requestID != nil {
+ logger = log.WithFields(log.Fields{
+ "request_id": *requestID,
+ })
+ }
+}
+
+func LogLevel() log.Level {
+ logLevelString := os.Getenv("LOG_LEVEL")
+
+ logLevel := log.InfoLevel
+
+ if logLevelString != "" {
+ logLevelString = strings.ToLower(logLevelString)
+
+ switch logLevelString {
+ case "error":
+ logLevel = log.ErrorLevel
+ case "warn":
+ logLevel = log.WarnLevel
+ case "info":
+ logLevel = log.InfoLevel
+ case "debug":
+ logLevel = log.DebugLevel
+ }
+ log.SetLevel(logLevel)
+ }
+ return logLevel
+}
+
+func getEnvironment() string {
+ environment := os.Getenv("ENVIRONMENT")
+ if environment == "" {
+ environment = "dev"
+ os.Setenv("ENVIRONMENT", "dev")
+ }
+ return environment
+}
+
+func getLogger() *log.Entry {
+ if logger == nil {
+ logger = log.WithFields(log.Fields{
+ "environment": getEnvironment(),
+ })
+ }
+
+ return logger
+}
+
+func InfoWithFields(fields map[string]interface{}, message interface{}) {
+ getLogger().WithFields(fields).Info(message)
+}
+
+func Info(format string, a ...interface{}) {
+ getLogger().Info(fmt.Sprintf(format, a...))
+}
+
+func ErrorWithFields(fields map[string]interface{}, err error) {
+ sendRaygunError(fields, err)
+ getLogger().WithFields(fields).Error(err)
+}
+
+func ErrorWithMsg(message string, err error) {
+ if err == nil {
+ err = errors.New(message)
+ }
+ ErrorWithFields(map[string]interface{}{
+ "message": message,
+ }, err)
+}
+
+func ErrorMsg(message string) {
+ ErrorWithMsg(message, nil)
+}
+
+func Warn(format string, a ...interface{}) {
+ getLogger().Warn(fmt.Sprintf(format, a...))
+}
+
+func WarnWithFields(fields map[string]interface{}, err error) {
+ getLogger().WithFields(fields).Warn(err)
+}
+
+func SQLDebugInfo(sql string) {
+ getLogger().WithFields(map[string]interface{}{
+ "sql": sql,
+ }).Debug("SQL query")
+}
+
+func LogShipmentID(id int64) {
+ InfoWithFields(map[string]interface{}{
+ "shipment_id": id,
+ }, "Current-shipment-ID")
+}
+
+func LogRequestInfo(req events.APIGatewayProxyRequest) {
+ InfoWithFields(map[string]interface{}{
+ "http_method": req.HTTPMethod,
+ "path": req.Path,
+ "api_gateway_request_id": req.RequestContext.RequestID,
+ "user_cognito_auth_provider": req.RequestContext.Identity.CognitoAuthenticationProvider,
+ "user_arn": req.RequestContext.Identity.UserArn,
+ }, "Request Info start")
+}
+
+func LogApiAudit(fields log.Fields) {
+ getLogger().WithFields(fields).Info("api-audit-log")
+}
+
+func LogSQSEvent(event events.SQSEvent) {
+ InfoWithFields(map[string]interface{}{
+ "records": event.Records,
+ }, "SQS event start")
+}
+
+func SetOutputToFile(file *os.File) {
+ log.SetOutput(file)
+}
+
+func ClearInfo() {
+ logger = nil
+}
+
+func sendRaygunError(fields map[string]interface{}, errToSend error) {
+ if isDebug || raygunClient == nil {
+ // Don't log raygun errors on debug
+ return
+ }
+
+ env := getEnvironment()
+
+ tags := []string{env}
+ raygunClient.Version(build)
+ tags = append(tags, build)
+
+ if apiRequest != nil {
+ methodAndPath := apiRequest.HTTPMethod + ": " + apiRequest.Path
+ tags = append(tags, methodAndPath)
+ fields["body"] = apiRequest.Body
+ fields["query"] = apiRequest.QueryStringParameters
+ fields["identity"] = apiRequest.RequestContext.Identity
+ }
+
+ raygunClient.Tags(tags)
+ if currentRequestID != nil {
+ fields["request_id"] = currentRequestID
+ }
+
+ fields["env"] = env
+ raygunClient.CustomData(fields)
+ raygunClient.Request(fakeHttpRequest())
+
+ if errToSend == nil {
+ errToSend = errors.New("")
+ }
+ err := raygunClient.SendError(errToSend)
+ if err != nil {
+ log.Println("Failed to send raygun error:", err.Error())
+ }
+}
+
+func fakeHttpRequest() *http.Request {
+ if apiRequest == nil {
+ return nil
+ }
+
+ requestURL := url.URL{
+ Path: apiRequest.Path,
+ Host: apiRequest.Headers["Host"],
+ }
+ request := http.Request{
+ Method: apiRequest.HTTPMethod,
+ URL: &requestURL,
+ Header: apiRequest.MultiValueHeaders,
+ }
+ return &request
+}
diff --git a/logs/logs_test.go b/logs/logs_test.go
new file mode 100644
index 0000000..78fb518
--- /dev/null
+++ b/logs/logs_test.go
@@ -0,0 +1,32 @@
+package logs_test
+
+import (
+ "os"
+ "testing"
+
+ "gitlab.com/uafrica/go-utils/errors"
+ "gitlab.com/uafrica/go-utils/logs"
+)
+
+func TestLogs(t *testing.T) {
+ //requestID := t.Name()
+ //event := events.APIGatewayProxyRequest{}
+ os.Setenv("DEBUGGING", "true")
+ //logs.InitLogs(&requestID, &event)
+
+ // formatter := log.TextFormatter{}
+ // log.SetFormatter(&formatter)
+
+ logs.InfoWithFields(map[string]interface{}{"a": 1, "b": 2}, "MyLogMessage1")
+ logs.Info("MyLogMessage2=%d,%d,%d", 1, 2, 3)
+ logs.ErrorWithFields(map[string]interface{}{"a": 4, "b": 5}, errors.Errorf("simple mistake"))
+ logs.ErrorWithMsg("Error Message", errors.Errorf("another simple mistake"))
+ logs.Warn("Warning about a=%s,%s,%s", "a", "b", "c")
+ logs.WarnWithFields(map[string]interface{}{"a": 4, "b": 5}, errors.Errorf("Cant believe it failed"))
+ logs.SQLDebugInfo("SELECT * from user")
+ //logs.LogRequestInfo(event)
+ //api_logs.LogSQSEvent(sqsEvent)
+
+ //api_logs.Errorf("Debugging %d!", 456)
+ //api_logs.Error("Info")
+}
diff --git a/logs/sensitive_words.go b/logs/sensitive_words.go
new file mode 100644
index 0000000..f525a35
--- /dev/null
+++ b/logs/sensitive_words.go
@@ -0,0 +1,71 @@
+package logs
+
+import (
+ "bytes"
+ "regexp"
+ "strings"
+)
+
+//word may only consist of alpha-numerics with delimeters inside,
+//e.g. OTP "12345" or Card number "1234-12345678-12345678-1234"
+//and the delimiters may be spaces, dashes, underscores, slashes and dots
+//and it is considered case insensitive
+
+const sensitiveWordPattern = `[a-z0-9]([a-z0-9\.\\\/ _-]*[a-z0-9])*`
+
+var sensitiveWordRegex = regexp.MustCompile("^" + sensitiveWordPattern + "$")
+
+const delimiters = "()[]{}!@#$%^&*-=_+;:'\"|\\/?<>,.~` \n\r"
+
+func FilterSensitiveWordsMap(s string, wordsMap map[string]bool) (filtered string, changed bool) {
+ if len(wordsMap) == 0 {
+ return s, false
+ }
+
+ changed = false
+ f := []byte(s)
+ for word := range wordsMap {
+ //it will be inefficient to compile regex for each word in each context
+ //much quicker to just look for the word and see if it is delimited as required
+ //not to mach short words as part of longer words which may expose the word be assumption
+ //e.g. OTP "202" should not match part of a date 2021-01-02 making it ***1-01-02
+ wLen := len(word)
+ offset := 0
+ fLen := len(f)
+ for offset < fLen {
+ index := bytes.Index(f[offset:], []byte(word)) + offset
+ if index < offset {
+ break //word not found
+ }
+
+ //found the word, check delimiters before/after
+ if index > 0 && strings.IndexByte(delimiters, f[index-1]) < 0 {
+ offset = index + 1 //word match without delimiter before
+ continue
+ }
+
+ if index+wLen < fLen && strings.IndexByte(delimiters, f[index+wLen]) < 0 {
+ offset = index + 1 //word match without delimiter after
+ continue
+ }
+
+ //has delimiter after, this is a word match, replace any length match with 3 stars "***"
+ //pad length if required
+ pad := 0
+ for fLen < index+3 {
+ f = append(f, ' ')
+ fLen++
+ pad++
+ }
+ f = append(f[:index+3], f[index+wLen:fLen-pad]...)
+ f[index] = '*'
+ f[index+1] = '*'
+ f[index+2] = '*'
+ fLen = len(f)
+ changed = true
+ offset = index + 3 //for loop skipped index over word, now skip offset over delimiter
+ }
+ }
+ filtered = string(f)
+ return
+}
diff --git a/logger/logger_test.go b/logs/sensitive_words_test.go
similarity index 93%
rename from logger/logger_test.go
rename to logs/sensitive_words_test.go
index cd3df52..53a51d0 100644
--- a/logger/logger_test.go
+++ b/logs/sensitive_words_test.go
@@ -1,14 +1,12 @@
-package logger_test
+package logs_test
import (
"testing"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
)
func TestFilter(t *testing.T) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
tests := []test{
{"Your new OTP is 12345", map[string]bool{"12345": true}, "Your new OTP is ***"}, //match at end of string
{"Your new OTP is 12345.", map[string]bool{"12345": true}, "Your new OTP is ***."}, //match in middle
@@ -36,7 +34,7 @@ func TestFilter(t *testing.T) {
{"1111.2222.3333.4444.5555.6666.7777.8888", map[string]bool{"8888": true}, "1111.2222.3333.4444.5555.6666.7777.***"}, //long replace at end
}
for testNr, test := range tests {
- filtered, changed := logger.FilterSensitiveWordsMap(test.Text, test.Words)
+ filtered, changed := logs.FilterSensitiveWordsMap(test.Text, test.Words)
if test.Filtered != filtered {
t.Fatalf("test[%d]: %s --%+v--> %s != %s", testNr, test.Text, test.Words, filtered, test.Filtered)
}
diff --git a/logger/stack.go b/logs/stack.go
similarity index 98%
rename from logger/stack.go
rename to logs/stack.go
index 0a178d0..05cfc09 100644
--- a/logger/stack.go
+++ b/logs/stack.go
@@ -1,4 +1,4 @@
-package logger
+package logs
import (
"bufio"
@@ -76,7 +76,7 @@ func CallStack() Stack {
ci := caller.Info()
if len(stack.Callers) == 0 {
if ci.Package == "runtime/debug" ||
- ci.Package == "gitlab.com/uafrica/go-utils/logger" ||
+ ci.Package == "gitlab.com/uafrica/go-utils/logs" ||
ci.Package == "gitlab.com/uafrica/go-utils/errors" ||
ci.Package == "" {
continue
diff --git a/number_utils/number_utils.go b/number_utils/number_utils.go
new file mode 100644
index 0000000..d183fe9
--- /dev/null
+++ b/number_utils/number_utils.go
@@ -0,0 +1,26 @@
+package number_utils
+
+import "math"
+
+func RoundFloat(value float64) float64 {
+ return math.Round(value*100) / 100 // 2 decimal places
+}
+
+func RoundFloatTo(value float64, to int) float64 {
+ toValue := math.Pow(10.0, float64(to))
+ return math.Round(value*toValue) / toValue
+}
+
+func UnwrapFloat64(f *float64) float64 {
+ if f == nil {
+ return 0.0
+ }
+ return *f
+}
+
+func UnwrapInt64(i *int64) int64 {
+ if i == nil {
+ return 0
+ }
+ return *i
+}
diff --git a/queues/README.md b/queues/README.md
deleted file mode 100644
index 1ac1212..0000000
--- a/queues/README.md
+++ /dev/null
@@ -1,162 +0,0 @@
-# Queus
-## Overview
-Queues are ```asyncronous``` [services](../service/README.md), meaning a service request is fired into a queue and processed when it gets to the front of the queue. There is no response from the service, but it could fail and be scheduled for another attempt after an optional delay.
-
-Asynchronous services should be used for any tasks that may take significant time to complete. The user could be offered API end-points to check on progress, get the result or cancel, as applicable.
-
-Queues are implemented with AWS SQS in production but replaced with in-memory channels in Go when doing local testing of your [API](../api/README.md). It is also possible to use AWS SQS while doing local API testing.
-
-## Contents
-* [Usage](#usage)
-* [Examples](#examples)
-
-## Usage
-The main function of your queue handler only runs in AWS SQS. For local testing, it is running inside the API handler. So you queue handler main function only creates an SQS service.
-
-1) Create the SQS service and specify:
- - the key used for request-id from event headers,
- - the event names and handler functions.
-
- Example:
-
- sqs.New("uafrica-request-id", sqsRoutes())
-
- The handler functions are defined as a map of event name and handler functions, e.g.:
-
- func sqsRoutes() map[string]interface{} {
- return map[string]interface{}{
- "pay-invoice": invoices.Pay,
- "send-email": emails.Send,
- }
- }
-
-2) Define Starters (optional)
-
- See [Service Starters](../service/README.MD#define-starters)
-
-3) Define Checks (optional)
-
- See [Service Checks](../service/README.MD#define-checks)
-
-4) Run() or ProcessFile()
-
- Now the service is defined it must Run() on a queue or one can test by processing one event from a file.
-
- We call Run() in production, or ProcessFile() for local testing.
-
-## Examples
-See [example1/sqs/sqs.go](../examples/core/sqs/main.go)
-
-# Handlers
-Each handler function referenced in your routes must have the following signature:
-
-```
-func MyHandler(ctx queues.Context, body MyBody) (err error) {
- ...
-}
-```
-
-Notes:
-* The route name is the name specified in
-
- ```ctx.NewEvent(...).Type(<name>)```
-* The ```body``` should have the same type you used elsewhere in
-
- ```ctx.NewEvent(...).Send(<type>)```
-
-* The optional body type Validate() method will be called and must return nil before the handler will be called.
-
-
-## Validate() error
-
-If body struct type implements Validate() error methods, they will be called before the handler is called. If either returns an error, the HTTP call fails with BadRequest with the error in the content and the handler will not be called.
-
-The Validate() method does not get a ctx, because it should only check the values inline and not do any db lookup or other service calls. Any such checks should be done inside the handler.
-
-You Validate() method may take a pointer receiver and fill in default values. If it takes a value received, it cannot change the value. For example, in this Validate() we set default limit to 10:
-
- func (p *GenericPageParams) Validate() error {
- if p.Limit == 0 {
- p.Limit = 10
- }
- if p.Limit <= 0 || p.Limit > 1000 {
- return errors.Errorf("invalid limit:%d (expecting 1..1000)", p.Limit)
- }
- return nil
- }
-
-## Params and Body Struct Types
-
-Both params and body types must be structs with public fields and json tags, e.g.:
-
- type MyParams struct {
- ID int64 `json:"id"`
- Offset int64 `json:"offset"`
- Limit int64 `json:"limit"`
- Name string `json:"name"`
- }
-
-The struct type is never marshaled to JSON by the framework, so omitempty has no effect.
-
-Pointers may be used for optional items, or you may treat a zero value to indicate absence.
-
-_Note: Body struct is populated with ```json.Unmarshal()``` but params has a manual parsing from the URL that does not implement all types. In generally we use strings and int64 for params. This may change in future releases._
-
-Arrays may be used for parameters, e.g.:
-
- type MyParams struct {
- IDs []int64 `json:"ids"`
- }
-
-Then a URL may specify either:
-* ```?ids=[1,2,3]```, or
-* ```?ids=1&ids=2&ids=3```
-
-You may include embedded structs in both params and body. Typical you may have a generic struct for paging with Offset and Limit, embed that into a get struct that adds an ID, and extend that for a handler to add more parameters, as in this example:
-
- type GenericPageParams struct {
- Limit int64 `json:"limit"`
- Offset int64 `json:"offset"`
- }
-
- type GenericGetParams struct {
- ID int64 `json:"id"`
- GenericPageParams
- }
-
- type MyGetParams struct {
- GenericGetParams
- Name string `json:"name"`
- }
-
-You should then call the embedded validation, else it will not happen:
-
- func (p MyGetParams) Validate() error {
- if p.Name == "" { return errors.Errorf("missing name") }
- if err := p.GenericGetParams.Validate(); err != nil { return err }
- return nil
- }
-
- func (p GenericGetParams) Validate() error {
- ...
- if err := p.GenericPageParams.Validate(); err != nil { return err }
- return nil
- }
-
-
-## Pointer or Value Type
-
-The Validate() method is always called with a pointer to the struct. So if your Validate() method takes a pointer receiver, it can change values. If it takes value receiver it can't. It is up to you, but recommendation is to use a pointer receiver ONLY when you do change the value in some cases, e.g. setting defaults that are not zero.
-
-The handler is always called with the struct values for params and body. The API creation in yuour main function will fail if you define any handler that takes a pointer type for either params of body.
-
-That also means your handler should not really change the param/body values. It can, as the changes will apply to the rest of the handler, e.g. if you do some db lookups to fill in some blanks in the struct. Just note that both params and body values are discarded when the handler returns. The framework will therefore log the original values passed to your handler after validation.
-
-
-## Errors
-
-When a handler cannot deal with an error, it returns an error that will be logged.
-
-# Async Event Retries
-
-Async services can send the same event to be retried after a delay, although retries are also supported in AWS SQS when the handler failed. Sending a retry inside the code gives us more control over varying retry schedules for different error cases... not yet sure what SQS offers and local channels offers no delay at present.
diff --git a/queues/event.go b/queues/event.go
deleted file mode 100644
index 18fca85..0000000
--- a/queues/event.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package queues
-
-import (
- "encoding/json"
- "fmt"
- "time"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
-)
-
-func NewEvent(producer Producer, queueName string) Event {
- if producer == nil {
- panic(errors.Errorf("NewEvent(producer=nil)"))
- }
-
- return Event{
- producer: producer,
- QueueName: queueName,
- TypeName: "",
- DueTime: time.Now(),
- RequestIDValue: "",
- ParamValues: map[string]string{},
- BodyJSON: "",
- }
-}
-
-type Event struct {
- producer Producer
- MessageID string //assigned by implementation (AWS/mem/..)
- QueueName string //queue determine sequencing, items in same queue are delivered one-after-the-other, other queues may deliver concurrent to this queue
- TypeName string //type determines which handler processes the event
- DueTime time.Time //do not process before this time
- RequestIDValue string //service request-id that sends the event - for tracing
- ParamValues map[string]string //parameters
- BodyJSON string //expecting a JSON string
-}
-
-func (event Event) Format(f fmt.State, c rune) {
- f.Write([]byte(fmt.Sprintf("{queue:%s,type:%s,due:%s,request-id:%s,params:%v,bodyJSON:%20.20s...,msg-id:%s}",
- event.QueueName,
- event.TypeName,
- event.DueTime.Format("2006-01-02 15:04:05"),
- event.RequestIDValue,
- event.ParamValues,
- event.BodyJSON,
- event.MessageID,
- )))
-}
-
-func (event Event) Delay(dur time.Duration) Event {
- if dur >= 0 {
- event.DueTime = time.Now().Add(dur)
- }
- return event
-}
-
-func (event Event) Type(typeName string) Event {
- if typeName != "" {
- event.TypeName = typeName
- }
- return event
-}
-
-func (event Event) RequestID(requestID string) Event {
- if requestID != "" {
- event.RequestIDValue = requestID
- }
- return event
-}
-
-func (event Event) Params(params map[string]string) Event {
- for n, v := range params {
- event.ParamValues[n] = v
- }
- return event
-}
-
-var log = logger.New()
-
-func (event Event) Send(value interface{}) (string, error) {
- if event.producer == nil {
- return "", errors.Errorf("send with producer==nil")
- }
- if value != nil {
- if valueString, ok := value.(string); ok {
- event.BodyJSON = valueString
- } else {
- jsonBody, err := json.Marshal(value)
- if err != nil {
- return "", errors.Wrapf(err, "failed to JSON encode event body")
- }
- event.BodyJSON = string(jsonBody)
- }
- }
- if msgID, err := event.producer.Send(event); err != nil {
- return "", errors.Wrapf(err, "failed to send event")
- } else {
- //do not log when we send to internal AUDIT/API_LOGS
- if event.QueueName != "AUDIT" && event.QueueName != "API_LOGS" {
- log.WithFields(map[string]interface{}{
- "queue": event.QueueName,
- "type": event.TypeName,
- "due": event.DueTime,
- "params": event.ParamValues,
- "body": event.BodyJSON,
- "msg_id": msgID,
- }).Info("Sent event")
- }
- return msgID, nil
- }
-}
diff --git a/queues/producer.go b/queues/producer.go
deleted file mode 100644
index 00df1db..0000000
--- a/queues/producer.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package queues
-
-//Producer sends an event for async processing
-type Producer interface {
- NewEvent(queueName string) Event
- Send(event Event) (msgID string, err error)
-}
diff --git a/queues/sqs_producer/README.md b/queues/sqs_producer/README.md
deleted file mode 100644
index d35d0ce..0000000
--- a/queues/sqs_producer/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-# AWS SQS Queues
-
-This is an implementation of go-utils/queues using AWS SQS.
\ No newline at end of file
diff --git a/queues/sqs_producer/producer.go b/queues/sqs_producer/producer.go
deleted file mode 100644
index 61c353d..0000000
--- a/queues/sqs_producer/producer.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package sqs_producer
-
-import (
- "os"
- "strings"
- "sync"
- "time"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/sqs"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
-)
-
-func New(requestIDHeaderKey string) queues.Producer {
- region := os.Getenv("AWS_REGION")
- if region == "" {
- panic(errors.Errorf("environment AWS_REGION is not defined"))
- }
- if requestIDHeaderKey == "" {
- requestIDHeaderKey = "request-id"
- }
- p := &producer{
- region: region,
- requestIDHeaderKey: requestIDHeaderKey,
- session: nil,
- queues: map[string]*QueueProducer{},
- }
- return p
-}
-
-type producer struct {
- sync.Mutex
- region string
- requestIDHeaderKey string
- session *session.Session
- queues map[string]*QueueProducer
-}
-
-func (producer *producer) NewEvent(queueName string) queues.Event {
- return queues.NewEvent(producer, queueName)
-}
-
-// Note: Calling code needs SQS IAM permissions
-func (producer *producer) Send(event queues.Event) (string, error) {
- logger.Debugf("SQS producer.Send(%+v)", event)
- messenger, ok := producer.queues[event.QueueName]
- if !ok {
- producer.Lock()
- defer producer.Unlock()
- messenger, ok = producer.queues[event.QueueName]
- if !ok {
- envName := strings.ToUpper(event.QueueName + "_QUEUE_URL")
- queueURL := os.Getenv(envName)
- if queueURL == "" {
- return "", errors.Errorf("cannot send to queue(%s) because environment(%s) is undefined", event.QueueName, envName)
- }
-
- // Make an AWS session
- sess, err := session.NewSessionWithOptions(session.Options{
- Config: aws.Config{
- Region: aws.String(producer.region),
- },
- })
- if err != nil {
- return "", errors.Wrapf(err, "failed to create AWS session")
- }
-
- messenger = &QueueProducer{
- producer: producer,
- session: sess,
- service: sqs.New(sess),
- queueURL: queueURL,
- }
- producer.queues[event.QueueName] = messenger
- } //if not defined in mutex
- } //if not defined
-
- if msgID, err := messenger.Send(event); err != nil {
- return "", errors.Wrapf(err, "failed to send")
- } else {
- return msgID, nil
- }
-}
-
-// QueueProducer sends an arbitrary message via SQS to a particular queue URL
-type QueueProducer struct {
- producer *producer
- session *session.Session
- service *sqs.SQS
- queueURL string
-}
-
-func (m *QueueProducer) Send(event queues.Event) (string, error) {
- //add params as message attributes
- msgAttrs := make(map[string]*sqs.MessageAttributeValue)
- for key, val := range event.ParamValues {
- msgAttrs[key] = &sqs.MessageAttributeValue{
- DataType: aws.String("String"),
- StringValue: aws.String(val),
- }
- }
-
- msgAttrs[m.producer.requestIDHeaderKey] = &sqs.MessageAttributeValue{
- DataType: aws.String("String"),
- StringValue: aws.String(event.RequestIDValue),
- }
- msgAttrs["type"] = &sqs.MessageAttributeValue{
- DataType: aws.String("String"),
- StringValue: aws.String(event.TypeName),
- }
-
- // SQS has max of 15 minutes delay
- // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
- // if due later than that, queue just for this much time
- delaySeconds := int64(time.Until(event.DueTime) / time.Second)
- if delaySeconds > 900 {
- delaySeconds = 900
- }
-
- if res, err := m.service.SendMessage(&sqs.SendMessageInput{
- MessageAttributes: msgAttrs,
- DelaySeconds: &delaySeconds,
- MessageBody: aws.String(event.BodyJSON),
- QueueUrl: &m.queueURL,
- }); err != nil {
- return "", errors.Wrapf(err, "failed to send")
- } else {
- return *res.MessageId, nil
- }
-}
diff --git a/redis/redis.go b/redis/redis.go
index 5dac499..ff44a4b 100644
--- a/redis/redis.go
+++ b/redis/redis.go
@@ -3,185 +3,149 @@ package redis
import (
"context"
"encoding/json"
- "os"
- "reflect"
+ "fmt"
+ "math"
+ "strings"
"time"
- "github.com/go-redis/redis/v8"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/string_utils"
+ "gitlab.com/uafrica/go-utils/logs"
+
+ "github.com/go-redis/redis_rate/v9"
+
+ "github.com/go-redis/redis/v8"
)
-type IRedis interface {
- string_utils.KeyReader
- Del(key string) error
- SetJSON(key string, value interface{}) error
- SetJSONIndefinitely(key string, value interface{}) error
- SetJSONForDur(key string, value interface{}, dur time.Duration) error
- GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool)
- SetString(key string, value string) error
- SetStringIndefinitely(key string, value string) error
- SetStringForDur(key string, value string, dur time.Duration) error
-}
+var ctx = context.Background()
-type redisWithContext struct {
- context.Context
- client *redis.Client
+type ClientWithHelpers struct {
+ Client *redis.Client
+ Available bool
}
-func New(ctx context.Context) (IRedis, error) {
- if globalClient == nil {
- var err error
- if globalClient, err = connect(); err != nil {
- return redisWithContext{Context: ctx}, errors.Wrapf(err, "cannot connect to REDIS")
- }
+func NewClient(addr string) *ClientWithHelpers {
+ return &ClientWithHelpers{
+ Client: redis.NewClient(&redis.Options{
+ MaxRetries: 1,
+ DialTimeout: time.Duration(1) * time.Second, //So max 2 second wait
+ Addr: addr,
+ Password: "", // no password set
+ DB: 0, // use default Db
+ }),
+ Available: true,
}
- return redisWithContext{
- Context: ctx,
- client: globalClient,
- }, nil
}
-func (r redisWithContext) Del(key string) error {
- if r.client == nil {
+func (r ClientWithHelpers) IsConnected() bool {
+ return r.Client != nil && r.Available == true
+}
+
+func (r ClientWithHelpers) DeleteByKey(key string) error {
+ if !r.IsConnected() {
return errors.Errorf("REDIS disabled: cannot del key(%s)", key)
}
- _, err := r.client.Del(r.Context, key).Result()
+ _, err := r.Client.Del(ctx, key).Result()
if err != nil {
return errors.Wrapf(err, "failed to del key(%s)", key)
}
- logger.Debugf("REDIS.Del(%s)", key)
return nil
}
-//set JSON value for 24h
-func (r redisWithContext) SetJSON(key string, value interface{}) error {
- return r.SetJSONForDur(key, value, 24*time.Hour)
-}
-
-func (r redisWithContext) SetJSONIndefinitely(key string, value interface{}) error {
- return r.SetJSONForDur(key, value, 0)
-}
-
-func (r redisWithContext) SetJSONForDur(key string, value interface{}, dur time.Duration) error {
- if r.client == nil {
- return errors.Errorf("REDIS disabled: cannot set JSON key(%s) = (%T)%v", key, value, value)
+func (r ClientWithHelpers) DeleteByKeyPattern(pattern string) {
+ if !r.IsConnected() {
+ return
}
- valueStr, ok := value.(string)
- if !ok {
- jsonBytes, err := json.Marshal(value)
+
+ iter := r.Client.Scan(ctx, 0, pattern, math.MaxInt64).Iterator()
+ for iter.Next(ctx) {
+ val := iter.Val()
+ err := r.Client.Del(ctx, val).Err()
if err != nil {
- return errors.Wrapf(err, "failed to JSON encode key(%s) = (%T)", key, value)
+ panic(err)
}
- valueStr = string(jsonBytes)
}
- if _, err := r.client.Set(r.Context, key, valueStr, dur).Result(); err != nil {
- return errors.Wrapf(err, "failed to set JSON key(%s)", key)
+ if err := iter.Err(); err != nil {
+ panic(err)
}
- logger.Debugf("REDIS.SetJSON(%s)=%s (%T) (exp: %v)", key, valueStr, value, dur)
- return nil
}
-//return:
-// nil,nil if key is not defined
-// nil,err if failed to get/determine if it exists, or failed to decode
-// <value>,nil if found and decoded
-func (r redisWithContext) GetJSON(key string, valueType reflect.Type) (value interface{}, ok bool) {
- if r.client == nil {
- return nil, false
+func (r ClientWithHelpers) SetObjectByKey(key string, object interface{}) {
+ if !r.IsConnected() {
+ return
}
- jsonValue, err := r.client.Get(r.Context, key).Result()
+
+ jsonBytes, err := json.Marshal(object)
if err != nil {
- return nil, false
- }
- newValuePtr := reflect.New(valueType)
- if err := json.Unmarshal([]byte(jsonValue), newValuePtr.Interface()); err != nil {
- return nil, false
+ logs.ErrorWithMsg("Error marshalling object to Redis: %s", err)
+ return
}
- return newValuePtr.Elem().Interface(), true
-}
-func (r redisWithContext) SetString(key string, value string) error {
- return r.SetStringForDur(key, value, 24*time.Hour)
-}
+ _, err = r.Client.Set(ctx, key, string(jsonBytes), 24*time.Hour).Result()
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err)
-func (r redisWithContext) SetStringIndefinitely(key string, value string) error {
- return r.SetStringForDur(key, value, 0)
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
+ }
+ }
}
-func (r redisWithContext) SetStringForDur(key string, value string, dur time.Duration) error {
- if r.client == nil {
- return errors.Errorf("REDIS disabled: cannot set key(%s) = (%T)%v", key, value, value)
+func (r ClientWithHelpers) SetObjectByKeyIndefinitely(key string, object interface{}) {
+ if !r.IsConnected() {
+ return
}
- if _, err := r.client.Set(r.Context, key, value, dur).Result(); err != nil {
- return errors.Wrapf(err, "failed to set key(%s)", key)
- }
- logger.Debugf("REDIS.SetString(%s)=%s (exp: %v)", key, value, dur)
- return nil
-}
-func (r redisWithContext) GetString(key string) (string, bool) {
- if r.client == nil {
- return "", false
+ jsonBytes, err := json.Marshal(object)
+ if err != nil {
+ logs.ErrorWithMsg("Error marshalling object to Redis", err)
+ return
}
- value, err := r.client.Get(r.Context, key).Result()
- if err != nil { /* Actual error */
- if err != redis.Nil { /* other than Key does not exist */
- logger.Errorf("Error fetching redis key(%s): %+v", key, err)
+
+ _, err = r.Client.Set(ctx, key, string(jsonBytes), 0).Result()
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Error setting value to Redis for key: %s", key), err)
+
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
- return "", false
}
- return value, true
+
}
-func (r redisWithContext) Keys(prefix string) []string {
- if r.client == nil {
- return nil
+func (r ClientWithHelpers) GetValueByKey(key string) string {
+ if !r.IsConnected() {
+ return ""
}
- value, err := r.client.Keys(r.Context, prefix+"*").Result()
- if err != nil { /* Actual error */
- if err != redis.Nil { /* other than no keys match */
- logger.Errorf("Error fetching redis keys(%s*): %+v", prefix, err)
- } else {
- logger.Errorf("Failed: %+v", err)
+
+ jsonString, err := r.Client.Get(ctx, key).Result()
+ if err == redis.Nil { /* Key does not exist */
+ return ""
+ } else if err != nil { /* Actual error */
+ logs.Warn(fmt.Sprintf("Error fetching object from Redis for key: %s", key), err)
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
- return nil //no matches
}
- logger.Debugf("Keys(%s): %+v", prefix, value)
- return value
+ return jsonString
}
-//global connection to REDIS used in all context
-var globalClient *redis.Client
-
-func connect() (*redis.Client, error) {
- host := os.Getenv("REDIS_HOST")
- if host == "false" {
- return nil, errors.Errorf("REDIS_HOST=false")
- }
+func (r ClientWithHelpers) RateLimit(key string, limitFn func(int) redis_rate.Limit, limit int) (bool, error) {
+ limiter := redis_rate.NewLimiter(r.Client)
+ res, err := limiter.Allow(ctx, key, limitFn(limit))
+ if err != nil {
+ logs.ErrorWithMsg(fmt.Sprintf("Redis Error rate limiting - %s", key), err)
- port := os.Getenv("REDIS_PORT")
- if os.Getenv("DEBUGGING") == "true" {
- host = "host.docker.internal"
- if os.Getenv("LOCAL") == "true" {
- host = "localhost"
- }
- env := os.Getenv("ENVIRONMENT")
- switch env {
- case "dev":
- port = "6380"
- case "stage":
- port = "6381"
- case "prod":
- port = "6383"
+ /* Prevent further calls in this execution from trying to connect and also timeout */
+ if strings.HasSuffix(err.Error(), "i/o timeout") {
+ r.Available = false
}
+
+ return false, err
+ } else {
+ logs.Info("Rate limiter - %s : %t with used %d, remaining %d", key, res.Allowed == 1, limit-res.Remaining, res.Remaining)
+ return res.Allowed == 1, nil
}
- logger.Debugf("Using REDIS(%s:%s)", host, port)
- globalClient = redis.NewClient(&redis.Options{
- Addr: host + ":" + port,
- Password: "", // no password set
- DB: 0, // use default DB
- })
- return globalClient, nil
-} //connect()
+}
diff --git a/redis/redis_test.go b/redis/redis_test.go
deleted file mode 100644
index f655889..0000000
--- a/redis/redis_test.go
+++ /dev/null
@@ -1,145 +0,0 @@
-package redis_test
-
-import (
- "context"
- "fmt"
- "os"
- "reflect"
- "testing"
- "time"
-
- "gitlab.com/uafrica/go-utils/redis"
-)
-
-func TestString(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- value := "abc123"
- if err := r.SetString(id, value); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetString(id); !ok {
- t.Fatalf("failed to get(%s)", id)
- } else {
- if v != value {
- t.Fatalf("%s=%s != %s", id, v, value)
- }
- }
-
- //must be able to delete
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s): %+v", id, err)
- }
-
- //delete non-existing must also succeed
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s) again: %+v", id, err)
- }
-
- //get after delete must indicate not exist
- if _, ok := r.GetString(id); ok {
- t.Fatalf("got(%s) after delete", id)
- }
-}
-
-func TestJSON(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- type Person struct {
- Name string `json:"name"`
- Surname string `json:"surname"`
- Count int `json:"count"`
- Dob time.Time `json:"dob"`
- }
- dob, err := time.Parse("2006-01-02", "1986-06-28")
- if err != nil {
- t.Fatalf("invalid dob: %+v", err)
- }
- value := Person{"Joe", "Blogs", 25, dob}
- if err := r.SetJSON(id, value); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); !ok {
- t.Fatalf("failed to get(%s): %+v", id, err)
- } else {
- if v != value {
- t.Fatalf("%s=%+v != %+v", id, v, value)
- }
- }
-
- //must be able to delete
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s): %+v", id, err)
- }
-
- //delete non-existing must also succeed
- if err := r.Del(id); err != nil {
- t.Fatalf("failed to del(%s) again: %+v", id, err)
- }
-
- //get after delete must indicate not exist
- if v, ok := r.GetJSON(id, reflect.TypeOf(Person{})); ok {
- t.Fatalf("got(%s) after delete: %+v", id, v)
- }
-}
-
-func TestExp(t *testing.T) {
- os.Setenv("REDIS_HOST", "localhost")
- os.Setenv("REDIS_PORT", "6380")
- ctx := context.Background()
- r, err := redis.New(ctx)
- if err != nil {
- t.Fatalf("failed to create redis: %+v", err)
- }
- id := fmt.Sprintf("%s_%v", t.Name(), time.Now().Unix())
- defer func() {
- r.Del(id)
- }()
-
- value := "abc123"
- if err := r.SetStringForDur(id, value, time.Second); err != nil {
- t.Fatalf("failed to set: (%T) %+v", err, err)
- }
-
- //get after set must return same value
- if v, ok := r.GetString(id); !ok {
- t.Fatalf("failed to get(%s)", id)
- } else {
- if v != value {
- t.Fatalf("%s=%s != %s", id, v, value)
- }
- }
-
- //wait 5 seconds
- t.Logf("waiting 5seconds for key to expire...")
- time.Sleep(time.Second * 5)
-
- //get after delete expire must fail
- if _, ok := r.GetString(id); ok {
- t.Fatalf("got(%s) after expiry", id)
- }
-}
diff --git a/reflection/reflection.go b/reflection/reflection.go
index b5ddd25..3f3458d 100644
--- a/reflection/reflection.go
+++ b/reflection/reflection.go
@@ -4,7 +4,7 @@ import (
"reflect"
"time"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
)
func SetPointerTime(field reflect.Value, value *time.Time) {
@@ -13,7 +13,7 @@ func SetPointerTime(field reflect.Value, value *time.Time) {
}
if field.Kind() != reflect.Ptr {
- logger.Error("Field need to be *Field")
+ logs.ErrorMsg("Field need to be *Field")
return
}
field.Set(reflect.ValueOf(value))
@@ -24,11 +24,11 @@ func SetInt64(field reflect.Value, value int64) {
return // Field doesn't exist
}
if field.Kind() != reflect.Int64 {
- logger.Error("Claims: Field is not of type Int64")
+ logs.ErrorMsg("Claims: Field is not of type Int64")
return
}
if field.OverflowInt(value) {
- logger.Error("Claims: Int overflow")
+ logs.ErrorMsg("Claims: Int overflow")
return
}
field.SetInt(value)
@@ -40,7 +40,7 @@ func SetPointerInt64(field reflect.Value, value *int64) {
}
if field.Kind() != reflect.Ptr {
- logger.Error("Field need to be *Int64")
+ logs.ErrorMsg("Field need to be *Int64")
return
}
@@ -52,7 +52,6 @@ func SetString(field reflect.Value, value string) {
return // Field doesn't exist
}
if field.Kind() != reflect.String {
- logger.Errorf("Claims: Field is not of type String: %v", field.Kind())
return
}
field.SetString(value)
@@ -90,3 +89,13 @@ func isFieldValid(field reflect.Value, readonly bool) bool {
return field.IsValid() && field.CanSet()
}
+
+func SetPointerString(field reflect.Value, value *string) {
+ if !isFieldValid(field, false) {
+ return // Field doesn't exist
+ }
+ if field.Kind() != reflect.Ptr {
+ return
+ }
+ field.Set(reflect.ValueOf(value))
+}
diff --git a/responses/responses.go b/responses/responses.go
new file mode 100644
index 0000000..6b6ecb4
--- /dev/null
+++ b/responses/responses.go
@@ -0,0 +1,121 @@
+package responses
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "gitlab.com/uafrica/go-utils/logs"
+ "gitlab.com/uafrica/go-utils/utils"
+
+ "github.com/aws/aws-lambda-go/events"
+)
+
+// =============================================================================
+// RESPONSE HELPER / CONVENIENCE FUNCTIONS
+// =============================================================================
+
+var ContentTypeJSONHeader = map[string]string{"Content-Type": "application/json"}
+
+// BadRequestResponse - invalid user input
+func BadRequestResponse(err error) (events.APIGatewayProxyResponse, error) {
+ return GenericResponse(http.StatusBadRequest, err.Error(), nil)
+}
+
+// NotFoundResponse - 404
+func NotFoundResponse(err error) (events.APIGatewayProxyResponse, error) {
+ return GenericResponse(http.StatusNotFound, err.Error(), nil)
+}
+
+func NoContent() (events.APIGatewayProxyResponse, error) {
+ return events.APIGatewayProxyResponse{
+ StatusCode: http.StatusNoContent,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ }, nil
+}
+
+func TooManyRequests(message string) (events.APIGatewayProxyResponse, error) {
+
+ response := map[string]string{
+ "message": message,
+ }
+ responseJson, err := json.Marshal(response)
+ if err != nil {
+ logs.Info("Could not create error message for ", message)
+ }
+
+ return events.APIGatewayProxyResponse{
+ StatusCode: http.StatusTooManyRequests,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ Body: string(responseJson),
+ }, nil
+}
+
+func LockedResponse(err error) (events.APIGatewayProxyResponse, error) {
+ return GenericResponse(http.StatusLocked, err.Error(), nil)
+}
+
+// InternalServerErrorResponse - something went wrong
+func InternalServerErrorResponse(err error) (events.APIGatewayProxyResponse, error) {
+ return GenericResponse(http.StatusInternalServerError, err.Error(), nil)
+}
+
+// SuccessResponse - everything worked
+func SuccessResponse(body string) (events.APIGatewayProxyResponse, error) {
+ return GenericResponse(http.StatusOK, body, nil)
+}
+
+// JSONSuccessResponse - try to JSONify any struct and send it as the body of a successful response
+// Return an internal server error if JSON marshalling fails
+// NOTE: struct fields must be exported (i.e. Capitalised) and tagged
+func JSONSuccessResponse(body interface{}) (events.APIGatewayProxyResponse, error) {
+ bodyBytes, err := json.Marshal(body)
+ if err != nil {
+ return InternalServerErrorResponse(err)
+ }
+
+ return GenericResponse(http.StatusOK, string(bodyBytes), ContentTypeJSONHeader)
+}
+
+// GenericResponse - called by the other functions
+func GenericResponse(statusCode int, body string, headers map[string]string) (events.APIGatewayProxyResponse, error) {
+ // Log the response
+ logs.Info("--> [APIGW] %d %s %s\n", statusCode, http.StatusText(statusCode), body)
+ // Return it
+
+ return events.APIGatewayProxyResponse{
+ StatusCode: statusCode,
+ Body: body + "\n",
+ Headers: utils.MergeMaps(utils.CorsHeaders(), headers),
+ }, nil
+}
+
+// SuccessResponse - everything worked
+func GenericSuccessResponse(message string) (events.APIGatewayProxyResponse, error) {
+ response := map[string]string{
+ "message": message,
+ }
+ return JSONSuccessResponse(response)
+}
+
+func MaintenanceResponse(message string) events.APIGatewayProxyResponse {
+ return events.APIGatewayProxyResponse{
+ StatusCode: http.StatusTeapot,
+ Body: message,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ }
+}
+
+func RateLimitResponse(message string) events.APIGatewayProxyResponse {
+ return events.APIGatewayProxyResponse{
+ StatusCode: http.StatusTooManyRequests,
+ Body: message,
+ Headers: utils.MergeMaps(utils.CorsHeaders(), ContentTypeJSONHeader),
+ }
+}
+
+func OptionsResponse() events.APIGatewayProxyResponse {
+ return events.APIGatewayProxyResponse{
+ StatusCode: http.StatusNoContent,
+ Headers: utils.MergeMaps(utils.CorsHeadersCached(), ContentTypeJSONHeader),
+ }
+}
diff --git a/s3/s3.go b/s3/s3.go
new file mode 100644
index 0000000..7bb8799
--- /dev/null
+++ b/s3/s3.go
@@ -0,0 +1,266 @@
+package s3
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws/session"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/google/uuid"
+)
+
+// S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request.
+type S3UploadResponse struct {
+ URL string `json:"url"`
+ Filename string `json:"filename"`
+ Bucket string `json:"bucket"`
+ FileSize int `json:"file_size"`
+}
+
+type MIMEType string
+
+const (
+ // MIMETypePDF defines the constant for the PDF MIME type.
+ MIMETypePDF MIMEType = "application/pdf"
+
+ // MIMETypeCSV defines the constant for the CSV MIME type.
+ MIMETypeCSV MIMEType = "text/csv"
+
+ // MIMETypeZIP defines the constant for the ZIP MIME type.
+ MIMETypeZIP MIMEType = "application/zip"
+
+ // MIMETypeJSON defines the constant for the JSON MIME type.
+ MIMETypeJSON MIMEType = "application/json"
+
+ // MIMETypeText defines the constant for the Plain text MIME type.
+ MIMETypeText MIMEType = "text/plain"
+
+ // MIMETypeImage defines the constant for the Image MIME type.
+ MIMETypeImage MIMEType = "image/*"
+
+ // MIMETypeDefault defines the constant for the default MIME type.
+ MIMETypeDefault MIMEType = "application/octet-stream"
+
+ // TypeXLS defines the constant for the XLS MIME type.
+ MIMETypeXLS MIMEType = "application/vnd.ms-excel"
+
+ // TypeXLSX defines the constant for the XLSX MIME type.
+ MIMETypeXLSX MIMEType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+
+)
+
+type SessionWithHelpers struct {
+ S3Session *s3.S3
+}
+
+func NewSession(session *session.Session) *SessionWithHelpers {
+ return &SessionWithHelpers{
+ S3Session: s3.New(session),
+ }
+}
+
+func (s SessionWithHelpers) Upload(data []byte, bucket, fileName string, metadata *map[string]*string, isDebug bool) (*s3.PutObjectOutput, error) {
+ mimeType := getTypeForFilename(fileName)
+ putInput := &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ ContentType: aws.String(string(mimeType)),
+ Body: bytes.NewReader(data),
+ }
+
+ if metadata != nil {
+ putInput.Metadata = *metadata
+ }
+
+ response, err := s.S3Session.PutObject(putInput)
+ if err != nil {
+ return nil, err
+ }
+
+ return response, nil
+}
+
+func (s SessionWithHelpers) UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, isDebug bool) (string, error) {
+ if mimeType == "" {
+ mimeType = getTypeForFilename(fileName)
+ }
+
+ expiry := time.Now().Add(24 * time.Hour)
+ putInput := &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ ContentType: aws.String(string(mimeType)),
+ Body: bytes.NewReader(data),
+ Expires: &expiry,
+ }
+
+ _, err := s.S3Session.PutObject(putInput)
+ if err != nil {
+ return "", err
+ }
+
+ return s.GetSignedDownloadURL(bucket, fileName, 24*time.Hour, isDebug)
+}
+
+// GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created.
+func (s SessionWithHelpers) GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, isDebug bool) (string, error) {
+ getInput := &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ getRequest, _ := s.S3Session.GetObjectRequest(getInput)
+
+ return getRequest.Presign(duration)
+}
+
+// UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse.
+func (s SessionWithHelpers) UploadWithFileExtension(data []byte, bucket, filePrefix string, fileExt string, mimeType MIMEType, isDebug bool) (*S3UploadResponse, error) {
+ fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt)
+
+ uploadUrl, err := s.UploadWithExpiry(data, bucket, fileName, mimeType, isDebug)
+ if err != nil {
+ return nil, err
+ }
+
+ fileSizeInBytes := binary.Size(data)
+
+ response := &S3UploadResponse{
+ URL: uploadUrl,
+ Filename: fileName,
+ Bucket: bucket,
+ FileSize: fileSizeInBytes,
+ }
+
+ return response, nil
+}
+
+func getTypeForFilename(f string) MIMEType {
+ ext := strings.ToLower(path.Ext(f))
+
+ switch ext {
+ case "pdf":
+ return MIMETypePDF
+ case "csv":
+ return MIMETypeCSV
+ case "zip":
+ return MIMETypeZIP
+ case "txt":
+ return MIMETypeText
+ }
+
+ return MIMETypeDefault
+}
+
+func (s SessionWithHelpers) GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) {
+ getInput := &s3.GetObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ getObjectOutput, err := s.S3Session.GetObject(getInput)
+ if err != nil {
+ return nil, err
+ }
+ return getObjectOutput, nil
+}
+
+func (s SessionWithHelpers) GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) {
+ headObjectInput := &s3.HeadObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ headObjectOutput, err := s.S3Session.HeadObject(headObjectInput)
+ if err != nil {
+ return nil, err
+ }
+ return headObjectOutput.Metadata, nil
+}
+
+// MoveObjectBucketToBucket - Move object from one S3 bucket to another
+func (s SessionWithHelpers) MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, isDebug bool) error {
+
+ err := s.CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, isDebug)
+ if err != nil {
+ return err
+ }
+
+ err = s.DeleteObjectFromBucket(sourceBucket, sourceFileName, isDebug)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CopyObjectBucketToBucket - Copy an object from one S3 bucket to another
+func (s SessionWithHelpers) CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, isDebug bool) error {
+ // copy the file
+ copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName)
+ copyObjectInput := &s3.CopyObjectInput{
+ Bucket: aws.String(destinationBucket), //destination bucket
+ CopySource: aws.String(copySource), //source path (ie: myBucket/myFile.csv)
+ Key: aws.String(destinationFilename), //filename on destination
+ }
+ _, err := s.S3Session.CopyObject(copyObjectInput)
+ if err != nil {
+ return err
+ }
+
+ // wait to see if the file copied successfully
+ err = s.S3Session.WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)})
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// DeleteObjectFromBucket - Delete an object from an S3 bucket
+func (s SessionWithHelpers) DeleteObjectFromBucket(bucket string, fileName string, isDebug bool) error {
+ // delete the file
+ deleteObjectInput := &s3.DeleteObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(fileName),
+ }
+ _, err := s.S3Session.DeleteObject(deleteObjectInput)
+ if err != nil {
+ return err
+ }
+
+ // wait to see if the file deleted successfully
+ err = s.S3Session.WaitUntilObjectNotExists(&s3.HeadObjectInput{
+ Bucket: aws.String(bucket), // the bucket we are deleting from
+ Key: aws.String(fileName), // the filename we are deleting
+ })
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func GetLogoURLFromFileName(fileName string) string {
+ logoUrl := "https://%s.s3.%s.amazonaws.com/logos/%s"
+ logoUrl = fmt.Sprintf(logoUrl, os.Getenv("S3_IMAGES_AND_NOTES_BUCKET"), os.Getenv("AWS_REGION"), fileName)
+
+ return logoUrl
+}
+
+func GetS3FileKey(fileName string, folder string) string {
+ // Trim leading and trailing slashes
+ fileName = strings.TrimLeft(fileName, "/")
+ fileName = strings.TrimRight(fileName, "/")
+
+ folder = strings.TrimLeft(folder, "/")
+ folder = strings.TrimRight(folder, "/")
+
+ return "/" + folder + "/" + fileName
+}
diff --git a/search/README.md b/search/README.md
index 8e1b934..b956a80 100644
--- a/search/README.md
+++ b/search/README.md
@@ -15,26 +15,4 @@ We use a document store and search to provide quick text searches from the API.
When a user is looking for an order, the API provides an end-point to search orders e.g. for "lcd screen", then the API does an OpenSearch query in the orders index, get N results and then read those orders from the orders table in the database (not OpenSearch) and return those results to the user.
-We therefore use OpenSearch only for searching and returning a list of document ids, then read the documents from the database. A document is typically an "order" but also anything else that we need to do free text searches on.
-
-## Testing
-The dev sub-directory contains a docker-compose.yml that runs OpenSearch loccally for test programs.
-Start it with:
-```
- cd dev
- docker-compose up -d
-```
-Then run the go test programs in this directory...
-E.g.:
-```go test -v --run TestLocalWriter```
-
-To work with this local instance from the command line:
-```curl --insecure -uadmin:admin "https://localhost:9200/_cat/indices"```
-
-If the test fail with index mapping error, you can delete the index before running the test, with the following command. It often happens when the code that generate the mapping changed and the existing index is incompatible with the new mapping:
-```
-curl --insecure -uadmin:admin -XDELETE "https://localhost:9200/go-utils-search-docs-test"
-```
-
-Some of the test programs also refer to the cloud instance created manually in V3, e.g. search_test.go TestDevWriter(). That can be updated or deleted as required.
-
+We therefore use OpenSearch only for searching and returning a list of document ids, then read the documents from the database. A document is typically an "order" but also anything else that we need to do free text searches on.
\ No newline at end of file
diff --git a/search/document_store.go b/search/document_store.go
index 233ef4e..07278e1 100644
--- a/search/document_store.go
+++ b/search/document_store.go
@@ -11,7 +11,7 @@ import (
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/reflection"
)
@@ -90,7 +90,7 @@ func (w *writer) DocumentStore(name string, tmpl interface{}) (DocumentStore, er
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal index mappings")
}
- logger.Infof("%s Index Mappings: %s", structType, string(ds.jsonMappings))
+ logs.Info("%s Index Mappings: %s", structType, string(ds.jsonMappings))
//define search response type
//similar to SearchResponseBody
@@ -164,7 +164,7 @@ func (ds *documentStore) Write(id string, data interface{}) error {
if res, err := ds.w.Write(indexName, id, data); err != nil {
return err
} else {
- logger.Debugf("IndexResponse: %+v", res)
+ logs.Info("IndexResponse: %+v", res)
}
return nil
}
@@ -217,17 +217,16 @@ func (ds *documentStore) Search(query Query, limit int64) (ids []string, totalCo
}
bodyData, _ := ioutil.ReadAll(searchResponse.Body)
- logger.Debugf("Response Body: %s", string(bodyData))
+ logs.Info("Response Body: %s", string(bodyData))
resBodyPtrValue := reflect.New(ds.searchResponseBodyType)
// if err = json.NewDecoder(searchResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
if err = json.Unmarshal(bodyData, resBodyPtrValue.Interface()); err != nil {
- logger.Errorf("search response body: %s", string(bodyData))
err = errors.Wrapf(err, "cannot decode search response body")
return
}
- logger.Debugf("Response Parsed: %+v", resBodyPtrValue.Interface())
+ logs.Info("Response Parsed: %+v", resBodyPtrValue.Interface())
hitsTotalValue, err := reflection.Get(resBodyPtrValue, ".hits.total.value")
if err != nil {
@@ -243,7 +242,7 @@ func (ds *documentStore) Search(query Query, limit int64) (ids []string, totalCo
err = errors.Wrapf(err, "cannot get search response documents")
return
}
- logger.Errorf("items: (%T) %+v", foundIDs.Interface(), foundIDs.Interface())
+ //logs.Errorf("items: (%T) %+v", foundIDs.Interface(), foundIDs.Interface())
return foundIDs.Interface().([]string), hitsTotalValue.Interface().(int), nil
}
diff --git a/search/document_store_test.go b/search/document_store_test.go
index 2c168d1..9e85738 100644
--- a/search/document_store_test.go
+++ b/search/document_store_test.go
@@ -8,7 +8,7 @@ import (
"time"
"github.com/google/uuid"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/search"
)
@@ -20,15 +20,15 @@ func TestLocalDocuments(t *testing.T) {
func TestDevDocuments(t *testing.T) {
testDocuments(t, search.Config{
- Addresses: []string{"https://search-uafrica-v3-api-logs-fefgiypvmb3sg5wqohgsbqnzvq.af-south-1.es.amazonaws.com/"}, //from AWS Console OpenSearch Service > Domains > uafrica-v3-api-logs > General Information: Domain Endpoints
+ Addresses: []string{"https://search-uafrica-v3-api-api_logs-fefgiypvmb3sg5wqohgsbqnzvq.af-south-1.es.amazonaws.com/"}, //from AWS Console OpenSearch Service > Domains > uafrica-v3-api-api_logs > General Information: Domain Endpoints
Username: "uafrica",
Password: "Aiz}a4ee",
})
}
func testDocuments(t *testing.T, c search.Config) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
+ logs.SetGlobalFormat(logs.NewConsole())
+ logs.SetGlobalLevel(logs.LevelDebug)
a, err := search.New(c)
if err != nil {
t.Fatalf("failed to create writer: %+v", err)
diff --git a/search/search_test.go b/search/search_test.go
index f3dd0be..dcd1e48 100644
--- a/search/search_test.go
+++ b/search/search_test.go
@@ -7,7 +7,7 @@ import (
"testing"
"time"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/search"
)
@@ -19,15 +19,15 @@ func TestLocalWriter(t *testing.T) {
func TestDevWriter(t *testing.T) {
test(t, search.Config{
- Addresses: []string{"https://search-uafrica-v3-api-logs-fefgiypvmb3sg5wqohgsbqnzvq.af-south-1.es.amazonaws.com/"}, //from AWS Console OpenSearch Service > Domains > uafrica-v3-api-logs > General Information: Domain Endpoints
+ Addresses: []string{"https://search-uafrica-v3-api-api_logs-fefgiypvmb3sg5wqohgsbqnzvq.af-south-1.es.amazonaws.com/"}, //from AWS Console OpenSearch Service > Domains > uafrica-v3-api-api_logs > General Information: Domain Endpoints
Username: "uafrica",
Password: "Aiz}a4ee",
})
}
func test(t *testing.T, c search.Config) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
+ logs.SetGlobalFormat(logs.NewConsole())
+ logs.SetGlobalLevel(logs.LevelDebug)
a, err := search.New(c)
if err != nil {
t.Fatalf("failed to create writer: %+v", err)
@@ -42,7 +42,7 @@ func test(t *testing.T, c search.Config) {
//write N records
methods := []string{"GET", "POST", "GET", "PATCH", "GET", "GET", "DELETE", "GET", "GET"} //more gets than others
paths := []string{"/users", "/orders", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates", "/accounts", "/shipment", "/rates"}
- N := 1
+ N := 100
testTime := time.Now().Add(-time.Hour * time.Duration(N))
for i := 0; i < N; i++ {
testTime = testTime.Add(time.Duration(float64(rand.Intn(100)) / 60.0 * float64(time.Hour)))
@@ -74,19 +74,17 @@ func test(t *testing.T, c search.Config) {
},
}
- docsByIDMap, totalCount, err := ts.Search(query, 10)
+ docs, totalCount, err := ts.Search(query, 10)
if err != nil {
t.Errorf("failed to search: %+v", err)
} else {
- t.Logf("search result total_count:%d with %d docs", totalCount, len(docsByIDMap))
- if len(docsByIDMap) > 10 {
- t.Errorf("got %d docs > max 10", len(docsByIDMap))
- }
- for id, doc := range docsByIDMap {
- t.Logf("id=%s doc=(%T)%+v", id, doc, doc)
- if _, ok := doc.(testStruct); !ok {
- t.Errorf("docs %T is not testStruct!", docsByIDMap)
+ if docsSlice, ok := docs.([]testStruct); ok {
+ t.Logf("search result total_count:%d with %d docs", totalCount, len(docsSlice))
+ if len(docsSlice) > 10 {
+ t.Errorf("got %d docs > max 10", len(docsSlice))
}
+ } else {
+ t.Errorf("docs %T is not []testStruct!", docs)
}
}
diff --git a/search/time_series.go b/search/time_series.go
index c819fb9..e9f87dd 100644
--- a/search/time_series.go
+++ b/search/time_series.go
@@ -12,7 +12,7 @@ import (
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/reflection"
)
@@ -27,25 +27,7 @@ type TimeSeriesHeader struct {
type TimeSeries interface {
Write(StartTime time.Time, EndTime time.Time, data interface{}) error
-
- //Search() returns docs indexed on OpenSearch document ID which cat be used in Get(id)
- //The docs value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
- //So you can safely type assert e.g.
- // type myType struct {...}
- // ts := search.TimeSeries(..., myType{})
- // docs,totalCount,err := ts.Search(...)
- // if err == nil {
- // for id,docValue := range docs {
- // doc := docValue.(myType)
- // ...
- // }
- // }
- Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error)
-
- //Get() takes the id returned in Search()
- //The id is uuid assigned by OpenSearch when documents are added with Write().
- //The document value type is the same as that of tmpl specified when you created the TimeSeries(..., tmpl)
- Get(id string) (interface{}, error)
+ Search(query Query, limit int64) (docs interface{}, totalCount int, err error)
}
type timeSeries struct {
@@ -59,14 +41,13 @@ type timeSeries struct {
createdDates map[string]bool
searchResponseBodyType reflect.Type
- getResponseBodyType reflect.Type
}
//purpose:
-// create a time series to write e.g. api logs
+// create a time series to write e.g. api api_logs
//parameters:
-// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-logs"
-// the actual indices in openSearch will be called "<indexName>-<ccyymmdd>" e.g. "uafrica-v3-api-logs-20210102"
+// name must be the openSearch index name prefix without the date, e.g. "uafrica-v3-api-api_logs"
+// the actual indices in openSearch will be called "<indexName>-<ccyymmdd>" e.g. "uafrica-v3-api-api_logs-20210102"
// tmpl must be your log data struct consisting of public fields as:
// Xxx string `json:"<name>" search:"keyword|text|long|date"` (can later add more types)
// Xxx time.Time `json:"<name>"` assumes type "date" for opensearch
@@ -121,7 +102,7 @@ func (w *writer) TimeSeries(name string, tmpl interface{}) (TimeSeries, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to marshal index mappings")
}
- logger.Infof("%s Index Mappings: %s", structType, string(ts.jsonMappings))
+ logs.Info("%s Index Mappings: %s", structType, string(ts.jsonMappings))
//define search response type
//similar to SearchResponseBody
@@ -133,18 +114,6 @@ func (w *writer) TimeSeries(name string, tmpl interface{}) (TimeSeries, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to make search response type for time-series")
}
-
- //define get response type
- //similar to GetResponseBody
- ts.getResponseBodyType, err = reflection.CloneType(
- reflect.TypeOf(GetResponseBody{}),
- map[string]reflect.Type{
- "._source": ts.dataType,
- })
- if err != nil {
- return nil, errors.Wrapf(err, "failed to make get response type for time-series")
- }
-
w.timeSeriesByName[name] = ts
return ts, nil
}
@@ -172,7 +141,7 @@ func structMappingProperties(structType reflect.Type) (map[string]MappingPropert
fieldName = jsonTags[0]
}
if fieldName == "" {
- logger.Debugf("Skip %s unnamed field %+v", structType, structField)
+ logs.Info("Skip %s unnamed field %+v", structType, structField)
continue
}
@@ -303,14 +272,14 @@ func (ts *timeSeries) Write(startTime, endTime time.Time, data interface{}) erro
if res, err := ts.w.Write(indexName, "", x.Elem().Interface()); err != nil {
return err
} else {
- logger.Debugf("IndexResponse: %+v", res)
+ logs.Info("IndexResponse: %+v", res)
}
return nil
}
//parameters:
-// indexName is index prefix before dash-date, e.g. "api-logs" then will look for "api-logs-<date>"
+// indexName is index prefix before dash-date, e.g. "api-api_logs" then will look for "api-api_logs-<date>"
//returns
// list of indices to delete with err==nil if deleted successfully
func (w *writer) DelOldTimeSeries(indexName string, olderThanDays int) ([]string, error) {
@@ -324,7 +293,7 @@ func (w *writer) DelOldTimeSeries(indexName string, olderThanDays int) ([]string
return nil, nil
}
- //make list of indices matching specified name e.g. "uafrica-v3-api-logs-*"
+ //make list of indices matching specified name e.g. "uafrica-v3-api-api_logs-*"
res, err := w.api.Indices.Get([]string{indexName + "-*"}, w.api.Indices.Get.WithHeader(map[string]string{"Accept": "application/json"}))
if err != nil {
return nil, errors.Wrapf(err, "failed to list existing %s-* indices", indexName)
@@ -351,10 +320,10 @@ func (w *writer) DelOldTimeSeries(indexName string, olderThanDays int) ([]string
for dailyIndexName, dailyIndexInfo := range indices {
dateStr := dailyIndexName[len(indexName)+1:]
if date, err := time.ParseInLocation("20060102", dateStr, t0.Location()); err != nil {
- logger.Debugf("Ignore index(%s) with invalid date(%s)", dailyIndexName, dateStr)
+ logs.Info("Ignore index(%s) with invalid date(%s)", dailyIndexName, dateStr)
} else {
if date.Before(timeThreshold) {
- logger.Debugf("Deleting index(%s).uuid(%s) older than %s days...", dailyIndexName, dailyIndexInfo.Settings.Index.UUID, timeThreshold)
+ logs.Info("Deleting index(%s).uuid(%s) older than %s days...", dailyIndexName, dailyIndexInfo.Settings.Index.UUID, timeThreshold)
indicesToDelete = append(indicesToDelete, dailyIndexName)
}
}
@@ -364,7 +333,7 @@ func (w *writer) DelOldTimeSeries(indexName string, olderThanDays int) ([]string
if err != nil {
return indicesToDelete, errors.Wrapf(err, "failed to delete indices(%s)", indicesToDelete)
}
- logger.Debugf("Deleted %d daily indices(%s) older than %d days", len(indicesToDelete), indicesToDelete, olderThanDays)
+ logs.Info("Deleted %d daily indices(%s) older than %d days", len(indicesToDelete), indicesToDelete, olderThanDays)
}
return indicesToDelete, nil
}
@@ -391,7 +360,7 @@ type IndexSettings struct {
//Search
//Return:
// docs will be a slice of the TimeSeries data type
-func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interface{}, totalCount int, err error) {
+func (ts *timeSeries) Search(query Query, limit int64) (docs interface{}, totalCount int, err error) {
if ts == nil {
return nil, 0, errors.Errorf("time series == nil")
}
@@ -416,7 +385,7 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interfac
}
jsonBody, _ := json.Marshal(body)
- logger.Debugf("Search: %s", string(jsonBody))
+ logs.Info("Search: %s", string(jsonBody))
search := opensearchapi.SearchRequest{
Index: []string{ts.name + "-*"},
Body: bytes.NewReader(jsonBody),
@@ -437,12 +406,12 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interfac
}
bodyData, _ := ioutil.ReadAll(searchResponse.Body)
- logger.Debugf("Response Body: %s", string(bodyData))
+ logs.Info("Response Body: %s", string(bodyData))
resBodyPtrValue := reflect.New(ts.searchResponseBodyType)
//if err = json.NewDecoder(searchResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
if err = json.Unmarshal(bodyData, resBodyPtrValue.Interface()); err != nil {
- logger.Errorf("search response body: %s", string(bodyData))
+ logs.Info("search response body: %s", string(bodyData))
err = errors.Wrapf(err, "cannot decode search response body")
return
}
@@ -456,66 +425,10 @@ func (ts *timeSeries) Search(query Query, limit int64) (docs map[string]interfac
return nil, 0, nil //no matches
}
- hits, err := reflection.Get(resBodyPtrValue, ".hits.hits[]")
+ items, err := reflection.Get(resBodyPtrValue, ".hits.hits[]._source")
if err != nil {
err = errors.Wrapf(err, "cannot get search response documents")
return
}
-
- docs = map[string]interface{}{}
- for i := 0; i < hits.Len(); i++ {
- hit := hits.Index(i)
- index := hit.Field(0).Interface().(string) //HitDoc.Index
- id := hit.Field(2).Interface().(string) //HitDoc.ID
- docs[index+"/"+id] = hit.Field(4).Interface() //HitDoc.Source
- }
- return docs, hitsTotalValue.Interface().(int), nil
-}
-
-func (ds *timeSeries) Get(indexSlashDocumentID string) (doc interface{}, err error) {
- if ds == nil {
- return nil, errors.Errorf("document store == nil")
- }
- parts := strings.SplitN(indexSlashDocumentID, "/", 2)
- get := opensearchapi.GetRequest{
- Index: parts[0],
- DocumentType: "_doc",
- DocumentID: parts[1],
- }
- getResponse, err := get.Do(context.Background(), ds.w.client)
- if err != nil {
- err = errors.Wrapf(err, "failed to get document")
- return
- }
-
- switch getResponse.StatusCode {
- case http.StatusOK:
- default:
- resBody, _ := ioutil.ReadAll(getResponse.Body)
- err = errors.Errorf("Get failed with HTTP status %v: %s", getResponse.StatusCode, string(resBody))
- return
- }
-
- resBodyPtrValue := reflect.New(ds.getResponseBodyType)
- if err = json.NewDecoder(getResponse.Body).Decode(resBodyPtrValue.Interface()); err != nil {
- err = errors.Wrapf(err, "cannot decode get response body")
- return
- }
-
- foundVar, err := reflection.Get(resBodyPtrValue, ".found")
- if err != nil {
- err = errors.Wrapf(err, "cannot get found value")
- return
- }
- if found, ok := foundVar.Interface().(bool); !ok || !found {
- return nil, nil //not found
- }
-
- //found
- source, err := reflection.Get(resBodyPtrValue, "._source")
- if err != nil {
- err = errors.Wrapf(err, "cannot get document from get response")
- return
- }
- return source.Interface(), nil
+ return items.Interface(), hitsTotalValue.Interface().(int), nil
}
diff --git a/search/writer.go b/search/writer.go
index 17a6dd0..46f4063 100644
--- a/search/writer.go
+++ b/search/writer.go
@@ -9,7 +9,7 @@ import (
opensearch "github.com/opensearch-project/opensearch-go"
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
)
type Writer interface {
@@ -43,7 +43,7 @@ func New(config Config) (Writer, error) {
return nil, errors.Wrapf(err, "cannot initialize opensearch connection")
}
// Print OpenSearch version information on console.
- logger.Debugf("Search client created with config: %+v", searchConfig)
+ logs.Info("Search client created with config: %+v", searchConfig)
w.api = opensearchapi.New(w.client)
return w, nil
diff --git a/secrets_manager/secrets_manager.go b/secrets_manager/secrets_manager.go
new file mode 100644
index 0000000..34220c1
--- /dev/null
+++ b/secrets_manager/secrets_manager.go
@@ -0,0 +1,124 @@
+package secrets_manager
+
+import (
+ "encoding/base64"
+ "fmt"
+ "os"
+
+ "gitlab.com/uafrica/go-utils/logs"
+ "gitlab.com/uafrica/go-utils/struct_utils"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/secretsmanager"
+ "github.com/aws/aws-secretsmanager-caching-go/secretcache"
+)
+
+type DatabaseCredentials struct {
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Engine string `json:"engine"`
+ Host string `json:"host"`
+ Port int `json:"port"`
+ InstanceIdentifier string `json:"dbInstanceIdentifier"`
+}
+
+var (
+ secretCache, _ = secretcache.New()
+ secretManagerRegion = "af-south-1"
+)
+
+func GetDatabaseCredentials(secretID string, isDebug bool) (DatabaseCredentials, error) {
+ secret, _ := getSecret(secretID, isDebug)
+ var credentials DatabaseCredentials
+ err := struct_utils.UnmarshalJSON([]byte(secret), &credentials)
+ if err != nil {
+ return DatabaseCredentials{}, err
+ }
+ return credentials, nil
+}
+
+func getSecret(secretID string, isDebug bool) (string, string) {
+ cachedSecret, err := secretCache.GetSecretString(secretID)
+ if err != nil {
+ logs.Info("Failed to get secret key from cache")
+ }
+ if cachedSecret != "" {
+ return cachedSecret, ""
+ }
+
+ awsSession := session.New()
+
+ // Get local config
+ if isDebug && os.Getenv("ENVIRONMENT") != "" {
+ env := os.Getenv("ENVIRONMENT")
+ awsSession = session.Must(session.NewSessionWithOptions(session.Options{
+ SharedConfigState: session.SharedConfigEnable,
+ Profile: fmt.Sprintf("shiplogic-%s", env),
+ }))
+ }
+
+ // Create a Secrets Manager client
+ svc := secretsmanager.New(awsSession, aws.NewConfig().WithRegion(secretManagerRegion))
+
+ input := &secretsmanager.GetSecretValueInput{
+ SecretId: aws.String(string(secretID)),
+ VersionStage: aws.String("AWSCURRENT"), // VersionStage defaults to AWSCURRENT if unspecified
+ }
+
+ // In this sample we only handle the specific exceptions for the 'GetSecretValue' API.
+ // See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
+
+ result, err := svc.GetSecretValue(input)
+ if err != nil {
+ if aerr, ok := err.(awserr.Error); ok {
+ switch aerr.Code() {
+ case secretsmanager.ErrCodeDecryptionFailure:
+ // Secrets Manager can't decrypt the protected secret text using the provided KMS key.
+ logs.Info(secretsmanager.ErrCodeDecryptionFailure, aerr.Error())
+
+ case secretsmanager.ErrCodeInternalServiceError:
+ // An error occurred on the server side.
+ logs.Info(secretsmanager.ErrCodeInternalServiceError, aerr.Error())
+
+ case secretsmanager.ErrCodeInvalidParameterException:
+ // You provided an invalid value for a parameter.
+ logs.Info(secretsmanager.ErrCodeInvalidParameterException, aerr.Error())
+
+ case secretsmanager.ErrCodeInvalidRequestException:
+ // You provided a parameter value that is not valid for the current state of the resource.
+ logs.Info(secretsmanager.ErrCodeInvalidRequestException, aerr.Error())
+
+ case secretsmanager.ErrCodeResourceNotFoundException:
+ // We can't find the resource that you asked for.
+ logs.Info("Can't find secret with ID: ", secretID)
+ logs.Info(secretsmanager.ErrCodeResourceNotFoundException, aerr.Error())
+ default:
+ logs.Info(err.Error())
+ }
+ } else {
+ // Print the error, cast err to awserr.Error to get the Code and
+ // Message from an error.
+ logs.Info(err.Error())
+ }
+ return "", ""
+ }
+
+ // Decrypts secret using the associated KMS CMK.
+ // Depending on whether the secret is a string or binary, one of these fields will be populated.
+ var secretString, decodedBinarySecret string
+ if result.SecretString != nil {
+ secretString = *result.SecretString
+ } else {
+ decodedBinarySecretBytes := make([]byte, base64.StdEncoding.DecodedLen(len(result.SecretBinary)))
+ length, err := base64.StdEncoding.Decode(decodedBinarySecretBytes, result.SecretBinary)
+ if err != nil {
+ logs.Info("Base64 Decode Error:", err)
+ return "", ""
+ }
+ decodedBinarySecret = string(decodedBinarySecretBytes[:length])
+ }
+
+ return secretString, decodedBinarySecret
+}
diff --git a/service/README.md b/service/README.md
deleted file mode 100644
index 6ee5841..0000000
--- a/service/README.md
+++ /dev/null
@@ -1,182 +0,0 @@
-# Package go-utils/services
-Services are classified as API, SQS, CRON or ADHOC:
-- [API](../api/README.md) services are synchronous, i.e. the user waits for the response,
-- [SQS](../queues/README.md) services are asynchronous, i.e. the request is queued and completes in good time without a response,
-- CRON services are scheduled to run at regular intervals with no request nor response, and
-- ADHOC services are executed on demand.
-
-Package go-utils/services defines the common layer among all the above services, and then there are packages specific to each of them to extend the service as needed.
-
-When you write and ```api```, you will import ```go-utils/api```.
-
-For ```cron```, import ```go-utils/cron```.
-
-For ```adhoc```, we have not yet created a package... TODO.
-
-For ```sqs``` it is a bit more complicated because we have two implementations, allowing the use of AWS SQS in production or simulating queues using golang channels when you debug. Still, you will import from ```go-utils/queues```.
-
-
-# Service Definition
-
-The following can be added to any type of service during the service definition, typically in your main function.
-
-## Starters
-
-Each starter will be called at the start of request processing, before your handler is called, and must succeed. If any of them returns and error, the service fails and your handler is not called.
-
-Starters are always called in the order they were defined.
-
-Starter are typically used to ensure you have a database and/or redis connection.
-
-Example:
-
- .WithStarter("db", db.Connector("core"))
-
-Where package db then defines:
-
- func Connector(dbName string) service.Starter {
- return &connector{
- dbName: dbName,
- dbConn: nil,
- }
- }
-
- type connector struct { ... }
-
- func (c *connector) Start(ctx service.Context) (interface{}, error) {
- if c.dbConn == nil {
- ...connect
- }
- return nil,nil
- }
-
-The starter does not have to return a value, but must return err==nil.
-If it does return a value, that value could be retrieve in a handler from the ctx using ctx.Get("db") which is the name specified in WithStarter().
-
-Use type assert on the value to get the correct type returned from the starter, e.g.:
-
- var dbConn *sql.Conn
- {
- db,_ := ctx.Get("db")
- dbConn = db.(*sql.Conn)
- }
- result, err := dbConn.Query(...)
-
-## Checks
-
-Checks are similar to starter, but called after all started completed, also in the order defined.
-
-Checks are typically used for applying rate limits or authorizastion.
-Since the starters completed by the time the check is called, the check may rely on starter values, e.g. it may use a db connection created by a starter.
-
-The only differences between service Starters and service Checks are:
-* starters are always called before checks (even if defined after), and
-* starters can return a value for handlers to retrieve which checks cannot do.
-
-Example:
-
- .WithCheck("claims", claims.ClaimsChecker{}).
- .WithCheck("rate", rates.Limiter(10))
-
-Each check implements service.ICheck and must return err==nil for processing to continue, or a descriptive reason for failing, e.g.:
-
- return errors.Errorf("rate limited for user(%s)", username)
-
-If your check is a pointer, the Check method gets a pointer receiver, and the check can set values in itself for subsequent calls, e.g. to count the nr of times it was called.
-
-Note that your check only lives in this instance and if it does rate limiting, the value should NOT be stored locally but rather in a central service such as REDIS.
-
-## Struct Types
-
-Struct types used for input params/body must be structs with public fields and json tags, e.g.:
-
- type MyParams struct {
- ID int64 `json:"id"`
- Offset int64 `json:"offset"`
- Limit int64 `json:"limit"`
- Name string `json:"name"`
- }
-
-The struct type is not marshaled to JSON by the framework, but is marshalled to a map that represents query params. The json omitempty decorator is then used to marshal or omit when not defined, and that may control behavior. One must include omitempty when absence/presence of an attribute is significant.
-
-Pointers may be used for optional items, or you may treat a zero value to indicate absence.
-
-## Validate()
-
-A ```Validate() error``` method is supported on all request and event structs such as API params and body and queued event body structs.
-
-If the struct type implements a ```Validate() error``` method, it is called after populating the struct and applying the claim, before the handler is called. If it returns an error, the service fails. In API, this will result in HTTP status BadRequest with the error in the content and the handler will not be called. In SQS it will log the error.
-
-The Validate() method does not get a ctx, because it should only check the values inline and not do any db lookup or other service calls. Any such checks should be done inside the handler.
-
-When Validate() method takes a pointer receiver it could fill in default values. If it takes a value received, changes to the struct are lost after the call to Validate(). For example, in this Validate() a default of 10 is set in Limit:
-
- func (p *GenericPageParams) Validate() error {
- if p.Limit == 0 {
- p.Limit = 10
- }
- if p.Limit <= 0 || p.Limit > 1000 {
- return errors.Errorf("invalid limit:%d (expecting 1..1000)", p.Limit)
- }
- return nil
- }
-
-When Validate() does not modify the struct, it should take a value receiver. There is no rule to enforce that, just good practice.
-
-## Pointer or Value
-
-The Validate() method is always called with a pointer to the struct. So if your Validate() method takes a pointer receiver, it can change values. If it takes value receiver it can't. It is up to you, but recommendation is to use a pointer receiver ONLY when you do change the value in some cases, e.g. setting defaults that are not zero.
-
-The handler is always called with the struct values for params and body. The API creation in yuour main function will fail if you define any handler that takes a pointer type for either params of body.
-
-That also means your handler should not really change the param/body values. It can, as the changes will apply to the rest of the handler, e.g. if you do some db lookups to fill in some blanks in the struct. Just note that both params and body values are discarded when the handler returns. The framework will therefore log the original values passed to your handler after validation.
-
-# Claims
-
-Claims identifies the user and typically consists of a user, account, role, permissions, etc...
-Claims are populated by an service.Check added to your API and should be set in the service context using ```service.Context.ClaimSet()``` method.
-
-Claim values may only be ```string``` or ```int64```. ```ClaimSet()``` will fail for other type of values.
-
-Claim values are identified by a field name which must be a public field name in Go, e.g. ```"AccountID"``` rather than ```"account_id"```. Once again, ```ClaimSet()``` will fail if not.
-
-If any of your param or body structs or sub-structs contains a field mathing a claim, they will be overwritten with the claim value before ```Validate()``` method is called. If your struct field match by name but use a type other than int64 or string, the handler will not be called with an error in claims.
-
-So:
-* Add claims to your API using ```.WithCheck("claim", myClaimChecker{})```
-* In ```myClaimChecker.Check()```:
- * Retrieve the claim, typically from a combination of:
- * The context (e.g. AWS values)
- * The HTTP header (e.g. some user id header)
- * The HTTP URL params (e.g. a token string)
- * The env (if you run in debug forcing some claim)
- * ...
- * Use your db/redis connections that was established in some ```api.WithStarter(...)``` to validate the claim or fail if not allowed
- * Set the with calls to ```ctx.SetClaim(...)```
- * Optionally, return a struct with all the claims, which handlers can retrieve using something like:
- ```claim,ok := ctx.Get("claims").(MyClaimStruct)```
-* All fields in params or body structs matching claim names will be overwritten by the time a handler is called.
-
-# Data Change Audits
-
-Data Change audit records are written with ctx.AuditChange()
-The AuditChange() method logs the changes between an original and new value.
-
-
-# Sending Async Events
-
-Events are sent for async processing with ```ctx.NewEvent()...Send()``` as in this example:
-
- if _, err := ctx.NewEvent(ctx, "BILLING").
- Type("provider-invoice").
- RequestID(ctx.RequestID()).
- Delay(time.Second * 5).
- Send(Invoice{ID: invoice.ID}); err != nil {
- ctx.Errorf("Failed to send event: %+v", err)
- }
-
-It is important to check the error, because it will fail if you specified invalid arguments or the producer fails to send for some reason such as authentication or the queue is not configured in the ENV etc...
-
-The 2nd argument in NewEvent is the queue name. There must be an environment variable defined with that ```<name>_QUEUE_URL=...```.
-
-The Delay() is optional and should only be used to delay processing of the event, e.g. to schedule a retry. The queue implementation may also impose restrictions on how long you can delay, e.g. AWS SQS may limits to 900 seconds.
diff --git a/service/context.go b/service/context.go
deleted file mode 100644
index 4b1f73b..0000000
--- a/service/context.go
+++ /dev/null
@@ -1,236 +0,0 @@
-package service
-
-import (
- "context"
- "reflect"
- "regexp"
- "time"
-
- "gitlab.com/uafrica/go-utils/audit"
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/string_utils"
-)
-
-// Ctx stores lambda-wide context e.g. claims, request ID etc.
-var Ctx Context
-
-type Context interface {
- context.Context
- logger.Logger
- queues.Producer
-
- RequestID() string
- MillisecondsSinceStart() int64
- StartTime() time.Time
-
- //set claim values - things that cannot change - typically UserID, AccountID, Username, ...
- //the fieldName must be public, i.e. start with uppercase A-Z, no underscores, its not a tag name :-)
- //once set, it will override any field in params/body struct recursively with this Golang field name (before validation)
- //Set will fail if the value is already set on this context, i.e. the value cannot change
- //Call this in your app Start(), before params/body is extracted
- //value may only be of type int64 or string for now...
- ClaimSet(fieldName string, value interface{}) error
-
- //Get can retrieve any claim value
- ClaimGet(fieldName string) (interface{}, bool)
-
- //Claim() return all values so you can iterate over them, but not change them...
- //note: a context.Context also support its own set of values, which you can use as you like
- //but the idea is that you use this instead, which we apply to params/body
- Claim() map[string]interface{}
-
- //context data (names must be snake_case)
- //unlike claim, any type of value may be stored
- //but like claims, value can never change
- WithValue(name string, value interface{}) Context
- Set(name string, value interface{}) error
- Get(name string) (interface{}, bool)
- //Value(name string) interface{} //return nil if not set, inherited from context.Context and overloaded to retrieve local first, just like Get()
- ValueOrDefault(name string, defaultValue interface{}) interface{}
- Data() map[string]interface{}
-
- //write a data change audit event
- AuditChange(eventType string, orgValue, newValue interface{})
-
- AddSensitiveWord(word string)
-}
-
-//values: are added to context and logger
-//these values are logged for every log event in this context
-//values can be added later using with value, but won't be logged
-// they are just for retrieval between unrelated packages, e.g.
-// authentication may set the user_id etc... and other package may retrieve it but not change it
-type valueKey string
-
-func (s service) NewContext(base context.Context, requestID string, values map[string]interface{}) (Context, error) {
- if values == nil {
- values = map[string]interface{}{}
- }
- values["request-id"] = requestID
-
- for n, v := range values {
- base = context.WithValue(base, valueKey(n), v)
- }
- l := logger.New().WithFields(values)
- //l.NextColor() - to be fixed but not showing colours in mage or cloud watch, so not urgent...
-
- Ctx = &serviceContext{
- Context: base,
- Logger: l,
- Producer: s.Producer,
- startTime: time.Now(),
- requestID: requestID,
- data: map[string]interface{}{},
- claim: map[string]interface{}{},
- }
-
- for starterName, starter := range s.starters {
- var starterData interface{}
- starterData, err := starter.Start(Ctx)
- if err != nil {
- Ctx.Errorf("Start(%s) failed: %+v ...", starterName, err)
- return nil, errors.Wrapf(err, "%s", starterName)
- }
- if err = Ctx.Set(starterName, starterData); err != nil {
- Ctx.Errorf("Start(%s) failed to set (%T)%+v: %+v ...", starterName, starterData, starterData, err)
- return nil, errors.Wrapf(err, "failed to set starter(%s) data=(%T)%+v", starterName, starterData, starterData)
- }
- Ctx.Debugf("Start(%s)=(%T)%+v", starterName, starterData, starterData)
- }
-
- return Ctx, nil
-}
-
-type serviceContext struct {
- context.Context
- logger.Logger
- queues.Producer
- startTime time.Time
- requestID string
- claim map[string]interface{}
- data map[string]interface{}
-}
-
-func (ctx serviceContext) RequestID() string {
- return ctx.requestID
-}
-
-const claimFieldNamePattern = `[A-Z][a-zA-Z0-9]*`
-
-var claimFieldNameRegex = regexp.MustCompile("^" + claimFieldNamePattern + "$")
-
-func (ctx *serviceContext) ClaimSet(fieldName string, value interface{}) error {
- if !claimFieldNameRegex.MatchString(fieldName) {
- return errors.Errorf("invalid claim field name \"%s\"", fieldName)
- }
- if oldValue, exists := ctx.claim[fieldName]; exists {
- return errors.Errorf("ClaimSet(%s=(%T)%v) failed because already set to (%T)%v", fieldName, value, value, oldValue, oldValue)
- }
- switch reflect.TypeOf(value).Kind() {
- case reflect.Int64:
- case reflect.String:
- default:
- panic(errors.Errorf("claim(%s)=(%T)%v is neither sting nor int64", fieldName, value, value))
- }
- ctx.claim[fieldName] = value
- return nil
-}
-
-func (ctx serviceContext) ClaimGet(fieldName string) (interface{}, bool) {
- if cv, ok := ctx.claim[fieldName]; ok {
- return cv, true
- }
- return nil, false
-}
-
-func (ctx serviceContext) Claim() map[string]interface{} {
- claim := map[string]interface{}{}
- for n, v := range ctx.claim {
- claim[n] = v
- }
- return claim
-}
-
-func (ctx *serviceContext) WithValue(name string, value interface{}) Context {
- ctx.Context = context.WithValue(ctx.Context, valueKey(name), value)
- return ctx
-}
-
-func (ctx *serviceContext) Set(name string, value interface{}) error {
- if !string_utils.IsSnakeCase(name) {
- return errors.Errorf("invalid context value name \"%s\"", name)
- }
- if oldValue, exists := ctx.data[name]; exists {
- return errors.Errorf("Set(%s=(%T)%v) failed because already set to (%T)%v", name, value, value, oldValue, oldValue)
- }
- ctx.data[name] = value
- return nil
-}
-
-func (ctx *serviceContext) Get(name string) (interface{}, bool) {
- if cv, ok := ctx.data[name]; ok {
- return cv, true
- }
- //alternative: try value from context.Context
- if cv := ctx.Context.Value(name); cv != nil {
- return cv, true
- }
- return nil, false
-}
-
-//Value override context.Context.Value to retrieve first from our own data
-func (ctx *serviceContext) Value(key interface{}) interface{} {
- if name, ok := key.(string); ok {
- if cv, ok := ctx.data[name]; ok {
- return cv
- }
- }
- //alternative: try value from context.Context
- if cv := ctx.Context.Value(key); cv != nil {
- return cv
- }
- return nil
-}
-
-func (ctx *serviceContext) Data() map[string]interface{} {
- data := map[string]interface{}{}
- for n, v := range ctx.data {
- data[n] = v
- }
- return data
-}
-
-func (ctx *serviceContext) MillisecondsSinceStart() int64 {
- return time.Since(ctx.startTime).Milliseconds()
-}
-
-func (ctx *serviceContext) StartTime() time.Time {
- return ctx.startTime
-}
-
-func (ctx *serviceContext) ValueOrDefault(name string, defaultValue interface{}) interface{} {
- if value := ctx.Value(valueKey(name)); value != nil {
- return value
- }
- return defaultValue
-}
-
-func (ctx *serviceContext) AuditChange(eventType string, orgValue, newValue interface{}) {
- username, _ := ctx.Claim()["username"].(string)
- if err := audit.SaveDataChange(
- ctx.requestID,
- username, //use username as source (will default to "SYSTEM" if undefined)
- eventType,
- orgValue,
- newValue,
- ); err != nil {
- ctx.Errorf("failed to save data change: %+v", err)
- return
- }
-}
-
-func (ctx *serviceContext) AddSensitiveWord(word string) {
- ctx.Logger = ctx.Logger.WithSensitiveWord(word)
-}
diff --git a/service/service.go b/service/service.go
deleted file mode 100644
index 3eda7b0..0000000
--- a/service/service.go
+++ /dev/null
@@ -1,77 +0,0 @@
-package service
-
-import (
- "context"
- "os"
-
- "gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
- "gitlab.com/uafrica/go-utils/queues"
- "gitlab.com/uafrica/go-utils/string_utils"
-)
-
-type Service interface {
- logger.Logger
- queues.Producer
- WithStarter(name string, starter Starter) Service
- WithProducer(producer queues.Producer) Service
- NewContext(base context.Context, requestID string, values map[string]interface{}) (Context, error)
-}
-
-func New() Service {
- env := os.Getenv("ENVIRONMENT") //todo: support config loading for local dev and env for lambda in prod
- if env == "" {
- env = "dev"
- }
- return service{
- Producer: nil,
- Logger: logger.New().WithFields(map[string]interface{}{"env": env}),
- env: env,
- starters: map[string]Starter{},
- }
-}
-
-type service struct {
- logger.Logger //for logging outside of context
- queues.Producer //for sending async events
- env string
- starters map[string]Starter
-}
-
-func (s service) Env() string {
- return s.env
-}
-
-//adds a starter function to call in each new context
-//they will be called in the sequence they were added (before api/cron/queue checks)
-//and they do not have details about the event
-//if starter returns error, processing fails
-//if starter succeeds, and return !=nil data, it is stored against the name
-// so your handler can retieve it with:
-// checkData := ctx.Value(name).(expectedType)
-// or
-// checkData,ok := ctx.Value(name).(expectedType)
-// if !ok { ... }
-//you can implement one starter that does everything and return a struct or
-//implement one for your db, one for rate limit, one for ...
-//the name must be snake-case, e.g. "this_is_my_starter_name"
-func (s service) WithStarter(name string, starter Starter) Service {
- if !string_utils.IsSnakeCase(name) {
- panic(errors.Errorf("invalid starter name=\"%s\", expecting snake_case names only", name))
- }
- if starter == nil {
- panic(errors.Errorf("starter(%s)==nil", name))
- }
- if _, ok := s.starters[name]; ok {
- panic(errors.Errorf("starter(%s) already defined", name))
- }
- s.starters[name] = starter
- return s
-}
-
-func (s service) WithProducer(producer queues.Producer) Service {
- if producer != nil {
- s.Producer = producer
- }
- return s
-}
diff --git a/service/start.go b/service/start.go
deleted file mode 100644
index 143a4e1..0000000
--- a/service/start.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package service
-
-type Starter interface {
- //called at the start of api/cron/queues processing, before checks, e.g. to ensure we have db connection
- //i.e. setup things that does not depend on the request/event details
- //if you need the request details, you need to implement a check for each of the api, cron and/or queue as needed, not a Start() method.
- Start(ctx Context) (interface{}, error)
-}
diff --git a/sqs/sqs.go b/sqs/sqs.go
new file mode 100644
index 0000000..0e19dc7
--- /dev/null
+++ b/sqs/sqs.go
@@ -0,0 +1,112 @@
+package sqs
+
+/*Package sqs provides a simple interface to send messages to AWS SQS*/
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/sqs"
+ "gitlab.com/uafrica/go-utils/logs"
+)
+
+// Messenger sends an arbitrary message via SQS
+type Messenger struct {
+ session *session.Session
+ service *sqs.SQS
+ queueURL string
+}
+
+// NewSQSMessenger constructs a Messenger which sends messages to an SQS queue
+// awsRegion - region that the queue was created
+// awsQueue - name of the queue
+// Note: Calling code needs SQS IAM permissions
+func NewSQSMessenger(awsRegion, queueUrl string) (*Messenger, error) {
+ // Make an AWS session
+ sess, err := session.NewSessionWithOptions(session.Options{
+ Config: aws.Config{
+ Region: aws.String(awsRegion),
+ },
+ })
+
+ if err != nil {
+ return nil, err
+ }
+
+ // Create SQS service
+ svc := sqs.New(sess)
+
+ return &Messenger{
+ session: sess,
+ service: svc,
+ queueURL: queueUrl,
+ }, nil
+}
+
+// SendSQSMessage sends a message to the queue associated with the messenger
+// headers - string message attributes of the SQS message (see AWS SQS documentation)
+// body - body of the SQS message (see AWS SQS documentation)
+func (m *Messenger) SendSQSMessage(headers map[string]string, body string, currentRequestID *string, sqsType string, headerKey string) (string, error) {
+ msgAttrs := make(map[string]*sqs.MessageAttributeValue)
+
+ for key, val := range headers {
+ msgAttrs[key] = &sqs.MessageAttributeValue{
+ DataType: aws.String("String"),
+ StringValue: aws.String(val),
+ }
+ }
+
+ // Add request ID
+ if currentRequestID != nil {
+ msgAttrs[headerKey] = &sqs.MessageAttributeValue{
+ DataType: aws.String("String"),
+ StringValue: aws.String(*currentRequestID),
+ }
+ }
+
+ msgAttrs["type"] = &sqs.MessageAttributeValue{
+ DataType: aws.String("String"),
+ StringValue: aws.String(sqsType),
+ }
+
+ res, err := m.service.SendMessage(&sqs.SendMessageInput{
+ MessageAttributes: msgAttrs,
+ MessageBody: aws.String(body),
+ QueueUrl: &m.queueURL,
+ })
+
+ if err != nil {
+ return "", err
+ }
+
+ return *res.MessageId, err
+}
+
+func SendSQSMessage(msgr *Messenger, region string, envQueueURLName string, objectToSend interface{}, currentRequestID *string, sqsType string, headerKey string) error {
+ if msgr == nil {
+ var err error
+ msgr, err = NewSQSMessenger(region, os.Getenv(envQueueURLName))
+ if err != nil {
+ logs.ErrorWithMsg("Failed to create sqs messenger with envQueueURLName: "+envQueueURLName, err)
+ }
+ }
+
+ jsonBytes, err := json.Marshal(objectToSend)
+ if err != nil {
+ logs.ErrorWithMsg("Failed to encode sqs event data", err)
+ return err
+ }
+
+ headers := map[string]string{"Name": "dummy"}
+ msgID, err := msgr.SendSQSMessage(headers, string(jsonBytes), currentRequestID, sqsType, headerKey)
+ if err != nil {
+ logs.ErrorWithMsg("Failed to send sqs event", err)
+ return err
+ }
+
+ logs.Info(fmt.Sprintf("Sent SQS message to %s with ID %s", envQueueURLName, msgID))
+ return nil
+}
diff --git a/string_utils/string_utils.go b/string_utils/string_utils.go
index 4250020..7dcaae8 100644
--- a/string_utils/string_utils.go
+++ b/string_utils/string_utils.go
@@ -133,6 +133,9 @@ func Int64ToString(number int64) string {
func IntToString(number int) string {
return strconv.Itoa(number)
}
+func StringToInt(stringValue string) (int, error) {
+ return strconv.Atoi(stringValue)
+}
func Int64SliceToString(numbers []int64) string {
numString := fmt.Sprint(numbers)
@@ -180,4 +183,38 @@ func IsEmpty(sp *string) bool {
}
return len(*sp) == 0
-}
\ No newline at end of file
+}
+
+// StringTrimQuotes - trims quotes from a string (ie: "foo" will return foo)
+func StringTrimQuotes(stringToTrim string) string {
+ if len(stringToTrim) >= 2 {
+ if stringToTrim[0] == '"' && stringToTrim[len(stringToTrim)-1] == '"' {
+ return stringToTrim[1 : len(stringToTrim)-1]
+ }
+ }
+
+ return stringToTrim
+}
+
+func KeyToHumanReadable(s string) string {
+ s = strings.TrimSpace(s)
+
+ re := regexp.MustCompile("(_|-)")
+ s = re.ReplaceAllString(s, " ")
+
+ return sentenceCase(string(s))
+}
+
+func sentenceCase(str string) string {
+ for i, v := range str {
+ return string(unicode.ToUpper(v)) + str[i+1:]
+ }
+ return ""
+}
+
+// RemoveUrlScheme Removes http:// or https:// from a URL
+func RemoveUrlScheme(str string) string {
+ newStr := strings.Replace(str, "http://", "", 1)
+ newStr = strings.Replace(str, "https://", "", 1)
+ return newStr
+}
diff --git a/struct_utils/named_values_to_struct.go b/struct_utils/named_values_to_struct.go
index ef07e61..68d8536 100644
--- a/struct_utils/named_values_to_struct.go
+++ b/struct_utils/named_values_to_struct.go
@@ -10,7 +10,7 @@ import (
"strings"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/string_utils"
)
@@ -49,10 +49,9 @@ func NamedValuesFromReader(prefix string, reader string_utils.KeyReader) map[str
value, ok := reader.GetString(key)
key = key[len(prefix):]
if !ok {
- logger.Debugf("Key(%s) undefined", key)
+ logs.Warn("Key(%s) undefined", key)
continue
}
- logger.Debugf("key(%s)=\"%s\"", key, value)
result[strings.ToLower(key)] = []string{value}
//split only if valid CSV between [...]
diff --git a/struct_utils/named_values_to_struct_test.go b/struct_utils/named_values_to_struct_test.go
index 533cb16..b048725 100644
--- a/struct_utils/named_values_to_struct_test.go
+++ b/struct_utils/named_values_to_struct_test.go
@@ -10,13 +10,13 @@ import (
"time"
"gitlab.com/uafrica/go-utils/errors"
- "gitlab.com/uafrica/go-utils/logger"
+ "gitlab.com/uafrica/go-utils/logs"
"gitlab.com/uafrica/go-utils/struct_utils"
)
func TestEnv(t *testing.T) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
+ logs.SetGlobalFormat(logs.NewConsole())
+ logs.SetGlobalLevel(logs.LevelDebug)
//booleans
os.Setenv("TEST_VALUE_ENABLE_CACHE", "true")
os.Setenv("TEST_VALUE_DISABLE_LOG", "true")
@@ -43,8 +43,8 @@ func TestEnv(t *testing.T) {
}
func TestURL1(t *testing.T) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
+ logs.SetGlobalFormat(logs.NewConsole())
+ logs.SetGlobalLevel(logs.LevelDebug)
queryParams := map[string]string{
"enable_cache": "true",
@@ -63,8 +63,8 @@ func TestURL1(t *testing.T) {
}
func TestURL2(t *testing.T) {
- logger.SetGlobalFormat(logger.NewConsole())
- logger.SetGlobalLevel(logger.LevelDebug)
+ logs.SetGlobalFormat(logs.NewConsole())
+ logs.SetGlobalLevel(logs.LevelDebug)
queryParams := map[string]string{
"disable_log": "true",
diff --git a/utils/utils.go b/utils/utils.go
new file mode 100644
index 0000000..d60ab90
--- /dev/null
+++ b/utils/utils.go
@@ -0,0 +1,166 @@
+package utils
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strconv"
+ "strings"
+
+ "gitlab.com/uafrica/go-utils/struct_utils"
+)
+
+// GetEnv is a helper function for getting environment variables with a default
+func GetEnv(name string, def string) (env string) {
+ // If variable not set, provide default
+ if env = os.Getenv(name); env == "" {
+ env = def
+ }
+ return
+}
+
+// CorsHeaders returns a map to allow Cors
+func CorsHeaders() map[string]string {
+ return map[string]string{
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Headers": "*",
+ "Access-Control-Allow-Methods": "OPTIONS,GET,PUT,POST,DELETE,PATCH,HEAD",
+ "Access-Control-Max-Age": "86400",
+ "Access-Control-Allow-Credentials": "true",
+ }
+}
+
+func CorsHeadersCached() map[string]string {
+ return map[string]string{
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Headers": "*",
+ "Access-Control-Allow-Methods": "OPTIONS,GET,PUT,POST,DELETE,PATCH,HEAD",
+ "Access-Control-Max-Age": "86400",
+ "Access-Control-Allow-Credentials": "true",
+ }
+}
+
+func ZipData(fileName string, data []byte) ([]byte, error) {
+ // Create zip
+ buf := new(bytes.Buffer)
+
+ // Create a new zip archive.
+ zipWriter := zip.NewWriter(buf)
+
+ // Write data
+ file, err := zipWriter.Create(fileName)
+ if err != nil {
+ return nil, err
+ }
+ _, err = file.Write(data)
+ if err != nil {
+ return nil, err
+ }
+
+ // Close zip
+ err = zipWriter.Close()
+ if err != nil {
+ return []byte(""), err
+ }
+
+ return buf.Bytes(), nil
+}
+
+func UnzipData(data []byte) (map[string][]byte, error) {
+ // Create a new zip reader.
+ zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return nil, err
+ }
+
+ // Read all the files from zip archive
+ var fileData = make(map[string][]byte)
+ for _, zipFile := range zipReader.File {
+ unzippedData, err := readZipFile(zipFile)
+ if err != nil {
+ continue
+ }
+
+ fileData[zipFile.FileHeader.Name] = unzippedData
+ }
+
+ return fileData, nil
+}
+
+func readZipFile(zipFile *zip.File) ([]byte, error) {
+ zipFileData, err := zipFile.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer zipFileData.Close()
+
+ return ioutil.ReadAll(zipFileData)
+}
+
+func DeepCopy(toValue interface{}, fromValue interface{}) (err error) {
+ valueBytes, err := json.Marshal(fromValue)
+ if err != nil {
+ return err
+ }
+ err = struct_utils.UnmarshalJSON(valueBytes, toValue)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func UnwrapBool(b *bool) bool {
+ if b == nil {
+ return false
+ }
+ return *b
+}
+
+// MapStringInterfaceToMapStringString converts a generic value typed map to a map with string values
+func MapStringInterfaceToMapStringString(inputMap map[string]interface{}) map[string]string {
+ query := make(map[string]string)
+ for mapKey, mapVal := range inputMap {
+ // Check if mapVal is a slice or a single value
+ switch mapValTyped := mapVal.(type) {
+ case []interface{}:
+ // Slice - convert each element individually
+ var mapValString []string
+
+ // Loop through each element in the slice and check the type
+ for _, sliceElem := range mapValTyped {
+ switch sliceElemTyped := sliceElem.(type) {
+ case string:
+ // Enclose strings in escaped quotations
+ mapValString = append(mapValString, fmt.Sprintf("\"%v\"", sliceElemTyped))
+ case float64:
+ // Use FormatFloat for least amount of precision.
+ mapValString = append(mapValString, strconv.FormatFloat(sliceElemTyped, 'f', -1, 64))
+ default:
+ // Convert to string
+ mapValString = append(mapValString, fmt.Sprintf("%v", sliceElemTyped))
+ }
+ }
+ // Join as a comma seperated array
+ query[mapKey] = "[" + strings.Join(mapValString, ",") + "]"
+ default:
+ // Single value - convert to string
+ query[mapKey] = fmt.Sprintf("%v", mapVal)
+ }
+ }
+
+ return query
+}
+
+// MergeMaps If there are similar properties in the maps, the last one will be used as the value
+func MergeMaps(maps ...map[string]string) map[string]string {
+ ret := map[string]string{}
+ for _, mapV := range maps {
+ for k, v := range mapV {
+ ret[k] = v
+ }
+ }
+ return ret
+}
--
GitLab