package s3 import ( "bytes" "encoding/binary" "fmt" "net/url" "path" "strings" "time" "github.com/aws/aws-sdk-go/aws/awserr" "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/google/uuid" ) // S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request. type S3UploadResponse struct { URL string `json:"url"` Filename string `json:"filename"` Bucket string `json:"bucket"` FileSize int `json:"file_size"` } type S3UploadSettings struct { MimeType MIMEType RetrieveSignedUrl bool ExpiryDuration *time.Duration // Used to set expiry datetime of download links. NB: does not affect deletion of object from S3 bucket. AddContentDisposition bool FileName string // Used to specify the file name (excluding path) for content disposition; if FilePath is not present, it is used as the object key as well. FilePath string // The complete file path (including name) used for the object key in S3. GenerateFileNameFromParts bool // Whether the file extension needs to be specified. If true, supply FilePrefix and FileExt. Adds a UUID between the prefix and the extension. FilePrefix string // Required when GenerateFileNameFromParts is true FileExt string // Required when GenerateFileNameFromParts is true } // Duration constants const ( S3ExpiryDuration1Day time.Duration = 24 * time.Hour S3ExpiryDuration7Days time.Duration = 7 * 24 * time.Hour ) type MIMEType string const ( // MIMETypePDF defines the constant for the PDF MIME type. MIMETypePDF MIMEType = "application/pdf" // MIMETypeCSV defines the constant for the CSV MIME type. MIMETypeCSV MIMEType = "text/csv" // MIMETypeZIP defines the constant for the ZIP MIME type. MIMETypeZIP MIMEType = "application/zip" // MIMETypeJSON defines the constant for the JSON MIME type. MIMETypeJSON MIMEType = "application/json" // MIMETypeText defines the constant for the Plain text MIME type. MIMETypeText MIMEType = "text/plain" // MIMETypeImage defines the constant for the Image MIME type. MIMETypeImage MIMEType = "image/*" // MIMETypePNG defines the constant for the PNG MIME type. MIMETypePNG MIMEType = "image/png" // MIMETypeDefault defines the constant for the default MIME type. MIMETypeDefault MIMEType = "application/octet-stream" // TypeXLS defines the constant for the XLS MIME type. MIMETypeXLS MIMEType = "application/vnd.ms-excel" // TypeXLSX defines the constant for the XLSX MIME type. MIMETypeXLSX MIMEType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) type SessionWithHelpers struct { S3Session *s3.S3 } func NewSession(session *session.Session) *SessionWithHelpers { return &SessionWithHelpers{ S3Session: s3.New(session), } } func (s SessionWithHelpers) Upload(data []byte, bucket, fileName string, metadata *map[string]*string) (*s3.PutObjectOutput, error) { mimeType := getTypeForFilename(fileName) putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), ContentType: aws.String(string(mimeType)), Body: bytes.NewReader(data), } if metadata != nil { putInput.Metadata = *metadata } response, err := s.S3Session.PutObject(putInput) if err != nil { return nil, err } return response, nil } func (s SessionWithHelpers) UploadWithSettings(data []byte, bucket, fileName string, settings S3UploadSettings) (string, error) { if settings.MimeType == "" { settings.MimeType = getTypeForFilename(fileName) } putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), ContentType: aws.String(string(settings.MimeType)), Body: bytes.NewReader(data), } // This sets the expiry date of the download link, not the deletion date of the object in the bucket. if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) putInput.Expires = &expiry } _, err := s.S3Session.PutObject(putInput) if err != nil { return "", err } if settings.RetrieveSignedUrl { var headers map[string]string fileNameHeader := fileName if settings.FileName != "" { fileNameHeader = settings.FileName } if settings.AddContentDisposition { headers = map[string]string{ "content-disposition": "attachment; filename=\"" + fileNameHeader + "\"", } } return s.GetSignedDownloadURL(bucket, fileName, 24*time.Hour, headers) } return "", nil } // UploadWithSettingsRevised can be renamed to UploadWithSettings once original function has been deprecated. func (s SessionWithHelpers) UploadWithSettingsRevised(data []byte, bucket string, settings S3UploadSettings) (S3UploadResponse, error) { var fileName, filePath, uploadUrl string // fileName is used as the object key in S3 if settings.FileName != "" { fileName = settings.FileName } if settings.GenerateFileNameFromParts { fileName = fmt.Sprintf("%s_%s.%s", settings.FilePrefix, uuid.New().String(), settings.FileExt) } if settings.FilePath != "" { filePath = settings.FilePath } else { filePath = fileName } // Uploaded objects require a key if fileName == "" { return S3UploadResponse{}, errors.Error("no file name supplied for upload") } if settings.MimeType == "" { settings.MimeType = getTypeForFilename(fileName) } putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(filePath), ContentType: aws.String(string(settings.MimeType)), Body: bytes.NewReader(data), } if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) putInput.Expires = &expiry } _, err := s.S3Session.PutObject(putInput) if err != nil { return S3UploadResponse{}, err } if settings.RetrieveSignedUrl { var headers map[string]string if settings.AddContentDisposition { headers = map[string]string{ "content-disposition": "attachment; filename=\"" + fileName + "\"", } } downloadUrlExpiry := 24 * time.Hour if settings.ExpiryDuration != nil { downloadUrlExpiry = *settings.ExpiryDuration } uploadUrl, err = s.GetSignedDownloadURL(bucket, filePath, downloadUrlExpiry, headers) if err != nil { return S3UploadResponse{}, err } } fileSizeInBytes := binary.Size(data) response := S3UploadResponse{ URL: uploadUrl, Filename: fileName, Bucket: bucket, FileSize: fileSizeInBytes, } return response, nil } func (s SessionWithHelpers) UploadWith1DayExpiry(data []byte, bucket, fileName string, mimeType MIMEType, shouldDownloadInsteadOfOpen bool) (string, error) { if mimeType == "" { mimeType = getTypeForFilename(fileName) } expiry := 24 * time.Hour signedUrl, err := s.UploadWithSettings(data, bucket, fileName, S3UploadSettings{ MimeType: mimeType, RetrieveSignedUrl: true, ExpiryDuration: &expiry, AddContentDisposition: shouldDownloadInsteadOfOpen, }) if err != nil { return "", err } return signedUrl, nil } // GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created. func (s SessionWithHelpers) GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, headers ...map[string]string) (string, error) { getInput := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } if headers != nil { if value, exists := headers[0]["content-disposition"]; exists { getInput.ResponseContentDisposition = &value } } getRequest, _ := s.S3Session.GetObjectRequest(getInput) fileExists, err := s.FileExists(bucket, fileName) if err != nil { return "", err } if !fileExists { return "", errors.Error("File does not exist") } return getRequest.Presign(duration) } func (s SessionWithHelpers) FileExists(bucket string, fileName string) (bool, error) { _, err := s.S3Session.HeadObject(&s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), }) if err != nil { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { case "NotFound": // s3.ErrCodeNoSuchKey does not work, aws is missing this error code so we hardwire a string return false, nil default: return false, err } } return false, err } return true, nil } // UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse. func (s SessionWithHelpers) UploadWithFileExtension(data []byte, bucket, filePrefix, fileExt string, mimeType MIMEType) (*S3UploadResponse, error) { fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt) duration := 24 * time.Hour uploadUrl, err := s.UploadWithSettings(data, bucket, fileName, S3UploadSettings{ MimeType: mimeType, RetrieveSignedUrl: true, ExpiryDuration: &duration, }) if err != nil { return nil, err } fileSizeInBytes := binary.Size(data) response := &S3UploadResponse{ URL: uploadUrl, Filename: fileName, Bucket: bucket, FileSize: fileSizeInBytes, } return response, nil } func getTypeForFilename(f string) MIMEType { ext := strings.ToLower(path.Ext(f)) switch ext { case "pdf": return MIMETypePDF case "csv": return MIMETypeCSV case "zip": return MIMETypeZIP case "txt": return MIMETypeText } return MIMETypeDefault } func (s SessionWithHelpers) GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) { getInput := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } getObjectOutput, err := s.S3Session.GetObject(getInput) if err != nil { return nil, err } return getObjectOutput, nil } func (s SessionWithHelpers) GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]*string, error) { headObjectInput := &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } headObjectOutput, err := s.S3Session.HeadObject(headObjectInput) if err != nil { return nil, err } return headObjectOutput.Metadata, nil } // MoveObjectBucketToBucket - Move object from one S3 bucket to another func (s SessionWithHelpers) MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, settings S3UploadSettings) error { err := s.CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, settings) if err != nil { return err } err = s.DeleteObjectFromBucket(sourceBucket, sourceFileName) if err != nil { return err } return nil } // CopyObjectBucketToBucket - Copy an object from one S3 bucket to another func (s SessionWithHelpers) CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, settings S3UploadSettings) error { // copy the file copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName) copyObjectInput := &s3.CopyObjectInput{ Bucket: aws.String(destinationBucket), // destination bucket CopySource: aws.String(copySource), // source path (ie: myBucket/myFile.csv) Key: aws.String(destinationFilename), // filename on destination } if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) copyObjectInput.Expires = &expiry } _, err := s.S3Session.CopyObject(copyObjectInput) if err != nil { return err } // wait to see if the file copied successfully err = s.S3Session.WaitUntilObjectExists(&s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)}) if err != nil { return err } return nil } // DeleteObjectFromBucket - Delete an object from an S3 bucket func (s SessionWithHelpers) DeleteObjectFromBucket(bucket string, fileName string) error { // delete the file deleteObjectInput := &s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } _, err := s.S3Session.DeleteObject(deleteObjectInput) if err != nil { return err } // wait to see if the file deleted successfully err = s.S3Session.WaitUntilObjectNotExists(&s3.HeadObjectInput{ Bucket: aws.String(bucket), // the bucket we are deleting from Key: aws.String(fileName), // the filename we are deleting }) if err != nil { return err } return nil } func GetS3FileKey(fileName string, folder string) string { var fileKey string // Trim leading and trailing slashes fileName = strings.TrimLeft(fileName, "/") fileName = strings.TrimRight(fileName, "/") if folder != "" { folder = strings.TrimLeft(folder, "/") folder = strings.TrimRight(folder, "/") fileKey += "/" + folder } fileKey += "/" + fileName return fileKey } func URLFromFileName(region string, bucket string, fileName string) string { logoUrl := "https://%s.s3.%s.amazonaws.com/%s" logoUrl = fmt.Sprintf(logoUrl, bucket, region, fileName) return logoUrl }