Skip to content
Snippets Groups Projects
Commit ac44bce0 authored by Cornel Rautenbach's avatar Cornel Rautenbach
Browse files

S3 to be more reusable

parent 22b236bd
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 38 additions and 69 deletions
s3_utils/s3_utils.go → s3/s3.go
+ 38
− 69
View file @ ac44bce0
package s3_utils
package s3
import (
"bytes"
......@@ -11,13 +11,10 @@ import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/google/uuid"
)
var s3Session *s3.S3
// S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request.
type S3UploadResponse struct {
URL string `json:"url"`
......@@ -29,16 +26,16 @@ type S3UploadResponse struct {
type MIMEType string
const (
// TypePDF defines the constant for the PDF MIME type.
// MIMETypePDF defines the constant for the PDF MIME type.
MIMETypePDF MIMEType = "application/pdf"
// TypeCSV defines the constant for the CSV MIME type.
// MIMETypeCSV defines the constant for the CSV MIME type.
MIMETypeCSV MIMEType = "text/csv"
// TypeZIP defines the constant for the ZIP MIME type.
// MIMETypeZIP defines the constant for the ZIP MIME type.
MIMETypeZIP MIMEType = "application/zip"
// TypeJSON defines the constant for the JSON MIME type.
// MIMETypeJSON defines the constant for the JSON MIME type.
MIMETypeJSON MIMEType = "application/json"
// MIMETypeText defines the constant for the Plain text MIME type.
......@@ -47,11 +44,21 @@ const (
// MIMETypeImage defines the constant for the Image MIME type.
MIMETypeImage MIMEType = "image/*"
// TypeDefault defines the constant for the default MIME type.
// MIMETypeDefault defines the constant for the default MIME type.
MIMETypeDefault MIMEType = "application/octet-stream"
)
func Upload(data []byte, bucket, fileName string, metadata *map[string]*string, isDebug bool) (*s3.PutObjectOutput, error) {
type SessionWithHelpers struct {
S3Session *s3.S3
}
func NewSession(session *s3.S3) *SessionWithHelpers {
return &SessionWithHelpers{
S3Session: session,
}
}
func (s SessionWithHelpers) Upload(data []byte, bucket, fileName string, metadata *map[string]*string, isDebug bool) (*s3.PutObjectOutput, error) {
mimeType := getTypeForFilename(fileName)
putInput := &s3.PutObjectInput{
Bucket: aws.String(bucket),
......@@ -64,7 +71,7 @@ func Upload(data []byte, bucket, fileName string, metadata *map[string]*string,
putInput.Metadata = *metadata
}
response, err := getSession(isDebug).PutObject(putInput)
response, err := s.S3Session.PutObject(putInput)
if err != nil {
return nil, err
}
......@@ -72,7 +79,7 @@ func Upload(data []byte, bucket, fileName string, metadata *map[string]*string,
return response, nil
}
func UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, isDebug bool) (string, error) {
func (s SessionWithHelpers) UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, isDebug bool) (string, error) {
if mimeType == "" {
mimeType = getTypeForFilename(fileName)
}
......@@ -86,30 +93,30 @@ func UploadWithExpiry(data []byte, bucket, fileName string, mimeType MIMEType, i
Expires: &expiry,
}
_, err := getSession(isDebug).PutObject(putInput)
_, err := s.S3Session.PutObject(putInput)
if err != nil {
return "", err
}
return GetSignedDownloadURL(bucket, fileName, 24*time.Hour, isDebug)
return s.GetSignedDownloadURL(bucket, fileName, 24*time.Hour, isDebug)
}
// GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created.
func GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, isDebug bool) (string, error) {
func (s SessionWithHelpers) GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, isDebug bool) (string, error) {
getInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}
getRequest, _ := getSession(isDebug).GetObjectRequest(getInput)
getRequest, _ := s.S3Session.GetObjectRequest(getInput)
return getRequest.Presign(duration)
}
// UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse.
func UploadWithFileExtension(data []byte, bucket, filePrefix string, fileExt string, mimeType MIMEType, isDebug bool) (*S3UploadResponse, error) {
func (s SessionWithHelpers) UploadWithFileExtension(data []byte, bucket, filePrefix string, fileExt string, mimeType MIMEType, isDebug bool) (*S3UploadResponse, error) {
fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt)
uploadUrl, err := UploadWithExpiry(data, bucket, fileName, mimeType, isDebug)
uploadUrl, err := s.UploadWithExpiry(data, bucket, fileName, mimeType, isDebug)
if err != nil {
return nil, err
}
......@@ -143,24 +150,24 @@ func getTypeForFilename(f string) MIMEType {
return MIMETypeDefault
}
func GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) {
func (s SessionWithHelpers) GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) {
getInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}
getObjectOutput, err := getSession(isDebug).GetObject(getInput)
getObjectOutput, err := s.S3Session.GetObject(getInput)
if err != nil {
return nil, err
}
return getObjectOutput, nil
}
func GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) {
func (s SessionWithHelpers) GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) {
headObjectInput := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}
headObjectOutput, err := getSession(isDebug).HeadObject(headObjectInput)
headObjectOutput, err := s.S3Session.HeadObject(headObjectInput)
if err != nil {
return nil, err
}
......@@ -168,14 +175,14 @@ func GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string
}
// MoveObjectBucketToBucket - Move object from one S3 bucket to another
func MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, isDebug bool) error {
func (s SessionWithHelpers) MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, isDebug bool) error {
err := CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, isDebug)
err := s.CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, isDebug)
if err != nil {
return err
}
err = DeleteObjectFromBucket(sourceBucket, sourceFileName, isDebug)
err = s.DeleteObjectFromBucket(sourceBucket, sourceFileName, isDebug)
if err != nil {
return err
}
......@@ -184,7 +191,7 @@ func MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sou
}
// CopyObjectBucketToBucket - Copy an object from one S3 bucket to another
func CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, isDebug bool) error {
func (s SessionWithHelpers) CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, isDebug bool) error {
// copy the file
copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName)
copyObjectInput := &s3.CopyObjectInput{
......@@ -192,13 +199,13 @@ func CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sou
CopySource: aws.String(copySource), //source path (ie: myBucket/myFile.csv)
Key: aws.String(destinationFilename), //filename on destination
}
_, err := getSession(isDebug).CopyObject(copyObjectInput)
_, err := s.S3Session.CopyObject(copyObjectInput)
if err != nil {
return err
}
// wait to see if the file copied successfully
err = getSession(isDebug).WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)})
err = s.S3Session.WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)})
if err != nil {
return err
}
......@@ -207,19 +214,19 @@ func CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sou
}
// DeleteObjectFromBucket - Delete an object from an S3 bucket
func DeleteObjectFromBucket(bucket string, fileName string, isDebug bool) error {
func (s SessionWithHelpers) DeleteObjectFromBucket(bucket string, fileName string, isDebug bool) error {
// delete the file
deleteObjectInput := &s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(fileName),
}
_, err := getSession(isDebug).DeleteObject(deleteObjectInput)
_, err := s.S3Session.DeleteObject(deleteObjectInput)
if err != nil {
return err
}
// wait to see if the file deleted successfully
err = getSession(isDebug).WaitUntilObjectNotExists(&s3.HeadObjectInput{
err = s.S3Session.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket), // the bucket we are deleting from
Key: aws.String(fileName), // the filename we are deleting
})
......@@ -247,41 +254,3 @@ func GetS3FileKey(fileName string, folder string) string {
return "/" + folder + "/" + fileName
}
func ListObjects(bucketName string, maxKeys int64, function func(*s3.ListObjectsOutput, bool) bool, isDebug bool) error {
listObjectsInput := &s3.ListObjectsInput{
Bucket: aws.String(bucketName),
MaxKeys: aws.Int64(maxKeys),
}
err := getSession(isDebug).ListObjectsPages(listObjectsInput, function)
if err != nil {
return err
}
return nil
}
func getSession(isDebug bool) *s3.S3 {
if s3Session == nil {
env := os.Getenv("ENVIRONMENT")
options := session.Options{
Config: aws.Config{
Region: aws.String("af-south-1"),
},
}
if isDebug {
options.Profile = fmt.Sprintf("shiplogic-%s", env)
}
sess, err := session.NewSessionWithOptions(options)
if err != nil {
return nil
}
s3Session = s3.New(sess)
}
return s3Session
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment