package s3 import ( "bytes" "context" "encoding/binary" "fmt" "net/url" "os" "path" "strings" "time" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/secrets_manager" std_errors "errors" "gitlab.bob.co.za/bob-public-utils/bobgroup-go-utils/errors" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/google/uuid" ) // S3UploadResponse defines the structure of a standard JSON response to a PDF/CSV/etc request. type S3UploadResponse struct { URL string `json:"url"` Filename string `json:"filename"` Bucket string `json:"bucket"` FileSize int `json:"file_size"` } type S3UploadSettings struct { MimeType MIMEType RetrieveSignedUrl bool ExpiryDuration *time.Duration // Used to set expiry datetime of download links. NB: does not affect deletion of object from S3 bucket. AddContentDisposition bool FilePath string FileName string FileExt string InsertUUID bool } // Duration constants const ( S3ExpiryDuration1Day time.Duration = 24 * time.Hour S3ExpiryDuration7Days time.Duration = 7 * 24 * time.Hour ) type MIMEType string const ( // MIMETypePDF defines the constant for the PDF MIME type. MIMETypePDF MIMEType = "application/pdf" // MIMETypeCSV defines the constant for the CSV MIME type. MIMETypeCSV MIMEType = "text/csv" // MIMETypeZIP defines the constant for the ZIP MIME type. MIMETypeZIP MIMEType = "application/zip" // MIMETypeJSON defines the constant for the JSON MIME type. MIMETypeJSON MIMEType = "application/json" // MIMETypeText defines the constant for the Plain text MIME type. MIMETypeText MIMEType = "text/plain" // MIMETypeImage defines the constant for the Image MIME type. MIMETypeImage MIMEType = "image/*" // MIMETypePNG defines the constant for the PNG MIME type. MIMETypePNG MIMEType = "image/png" // MIMETypeDefault defines the constant for the default MIME type. MIMETypeDefault MIMEType = "application/octet-stream" // TypeXLS defines the constant for the XLS MIME type. MIMETypeXLS MIMEType = "application/vnd.ms-excel" // TypeXLSX defines the constant for the XLSX MIME type. MIMETypeXLSX MIMEType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) var ( clients = map[string]*ClientWithHelpers{} ) type ClientWithHelpers struct { S3Client *s3.Client } func GetClient(isDebug bool, region ...string) *ClientWithHelpers { s3Region := os.Getenv("AWS_REGION") // Set custom region if region != nil && len(region) > 0 { s3Region = region[0] } // Check if client exists for region, if it does return it if s3Client, ok := clients[s3Region]; ok { return s3Client } cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(s3Region), config.WithCredentialsProvider(GetS3CredentialsProvider(isDebug)), ) if err != nil { return nil } s3Client := NewClient(cfg) clients[s3Region] = s3Client return s3Client } func GetS3CredentialsProvider(isDebug bool) credentials.StaticCredentialsProvider { secretID := os.Getenv("S3_SECRET_ID") s3CredentialsProvider, err := secrets_manager.GetS3UploadCredentialsProvider(secretID, isDebug) if err != nil { return credentials.StaticCredentialsProvider{} } return s3CredentialsProvider } func NewClient(cfg aws.Config) *ClientWithHelpers { return &ClientWithHelpers{ S3Client: s3.NewFromConfig(cfg), } } func (s ClientWithHelpers) Upload(data []byte, bucket, fileName string, metadata *map[string]string) (*s3.PutObjectOutput, error) { mimeType := getTypeForFilename(fileName) putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), ContentType: aws.String(string(mimeType)), Body: bytes.NewReader(data), } if metadata != nil { putInput.Metadata = *metadata } response, err := s.S3Client.PutObject(context.TODO(), putInput) if err != nil { return nil, err } return response, nil } func (s ClientWithHelpers) UploadWithSettings(data []byte, bucket, fileName string, settings S3UploadSettings) (string, error) { if settings.MimeType == "" { settings.MimeType = getTypeForFilename(fileName) } putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), ContentType: aws.String(string(settings.MimeType)), Body: bytes.NewReader(data), } // This sets the expiry date of the download link, not the deletion date of the object in the bucket. if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) putInput.Expires = &expiry } _, err := s.S3Client.PutObject(context.TODO(), putInput) if err != nil { return "", err } if settings.RetrieveSignedUrl { var headers map[string]string fileNameHeader := fileName if settings.FileName != "" { fileNameHeader = settings.FileName } if settings.AddContentDisposition { headers = map[string]string{ "content-disposition": "attachment; filename=\"" + fileNameHeader + "\"", } } return s.GetSignedDownloadURL(bucket, fileName, 24*time.Hour, headers) } return "", nil } // UploadWithSettingsRevised can be renamed to UploadWithSettings once original function has been deprecated. func (s ClientWithHelpers) UploadWithSettingsRevised(data []byte, bucket string, settings S3UploadSettings) (S3UploadResponse, error) { var fullFileName, uploadUrl string uuidString := "" if settings.InsertUUID { uuidString = fmt.Sprintf("_%s", uuid.New().String()) } if len(settings.FileExt) > 0 { if settings.FileExt[0] != '.' { settings.FileExt = fmt.Sprintf(".%s", settings.FileExt) } } if len(settings.FilePath) > 0 { if settings.FilePath[len(settings.FilePath)-1] != '/' { settings.FilePath = fmt.Sprintf("%s/", settings.FilePath) } } fullFileName = fmt.Sprintf("%s%s%s%s", settings.FilePath, settings.FileName, uuidString, settings.FileExt) // Uploaded objects require a key if fullFileName == "" { return S3UploadResponse{}, errors.Error("no file name supplied for upload") } if settings.MimeType == "" { settings.MimeType = getTypeForFilename(fullFileName) } putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fullFileName), ContentType: aws.String(string(settings.MimeType)), Body: bytes.NewReader(data), } if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) putInput.Expires = &expiry } _, err := s.S3Client.PutObject(context.TODO(), putInput) if err != nil { return S3UploadResponse{}, err } if settings.RetrieveSignedUrl { var headers map[string]string if settings.AddContentDisposition { headers = map[string]string{ "content-disposition": fmt.Sprintf("attachment; filename=\"%s%s\"", settings.FileName, settings.FileExt), } } downloadUrlExpiry := 24 * time.Hour if settings.ExpiryDuration != nil { downloadUrlExpiry = *settings.ExpiryDuration } uploadUrl, err = s.GetSignedDownloadURL(bucket, fullFileName, downloadUrlExpiry, headers) if err != nil { return S3UploadResponse{}, err } } fileSizeInBytes := binary.Size(data) response := S3UploadResponse{ URL: uploadUrl, Filename: fullFileName, Bucket: bucket, FileSize: fileSizeInBytes, } return response, nil } func (s ClientWithHelpers) UploadWith1DayExpiry(data []byte, bucket, fileName string, mimeType MIMEType, shouldDownloadInsteadOfOpen bool) (string, error) { if mimeType == "" { mimeType = getTypeForFilename(fileName) } expiry := 24 * time.Hour signedUrl, err := s.UploadWithSettings(data, bucket, fileName, S3UploadSettings{ MimeType: mimeType, RetrieveSignedUrl: true, ExpiryDuration: &expiry, AddContentDisposition: shouldDownloadInsteadOfOpen, }) if err != nil { return "", err } return signedUrl, nil } // GetSignedDownloadURL gets a signed download URL for the duration. If scv is nil, a new session will be created. func (s ClientWithHelpers) GetSignedDownloadURL(bucket string, fileName string, duration time.Duration, headers ...map[string]string) (string, error) { fileExists, err := s.FileExists(bucket, fileName) if err != nil { return "", err } if !fileExists { return "", errors.Error("File does not exist") } getInput := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } if headers != nil { if value, exists := headers[0]["content-disposition"]; exists { getInput.ResponseContentDisposition = &value } } presignClient := s3.NewPresignClient(s.S3Client) getRequest, err := presignClient.PresignGetObject(context.TODO(), getInput, func(po *s3.PresignOptions) { po.Expires = duration }) return getRequest.URL, err } // GetSignedUploadURL gets a signed upload URL for the duration. If scv is nil, a new session will be created. func (s ClientWithHelpers) GetSignedUploadURL(bucket string, fileName string, duration time.Duration) (string, error) { putInput := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } presignClient := s3.NewPresignClient(s.S3Client) putRequest, err := presignClient.PresignPutObject(context.TODO(), putInput, func(po *s3.PresignOptions) { po.Expires = duration }) return putRequest.URL, err } func (s ClientWithHelpers) FileExists(bucket string, fileName string) (bool, error) { _, err := s.S3Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), }) if err != nil { var notFoundErr *types.NotFound if std_errors.As(err, ¬FoundErr) { return false, nil } return false, err } return true, nil } // UploadWithFileExtension will upload a file to S3 and return a standard S3UploadResponse. func (s ClientWithHelpers) UploadWithFileExtension(data []byte, bucket, filePrefix, fileExt string, mimeType MIMEType) (*S3UploadResponse, error) { fileName := fmt.Sprintf("%s_%s.%s", filePrefix, uuid.New().String(), fileExt) duration := 24 * time.Hour uploadUrl, err := s.UploadWithSettings(data, bucket, fileName, S3UploadSettings{ MimeType: mimeType, RetrieveSignedUrl: true, ExpiryDuration: &duration, }) if err != nil { return nil, err } fileSizeInBytes := binary.Size(data) response := &S3UploadResponse{ URL: uploadUrl, Filename: fileName, Bucket: bucket, FileSize: fileSizeInBytes, } return response, nil } func getTypeForFilename(f string) MIMEType { ext := strings.ToLower(path.Ext(f)) switch ext { case "pdf": return MIMETypePDF case "csv": return MIMETypeCSV case "zip": return MIMETypeZIP case "txt": return MIMETypeText } return MIMETypeDefault } func (s ClientWithHelpers) GetObject(bucket string, fileName string, isDebug bool) (*s3.GetObjectOutput, error) { getInput := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } getObjectOutput, err := s.S3Client.GetObject(context.TODO(), getInput) if err != nil { return nil, err } return getObjectOutput, nil } func (s ClientWithHelpers) GetObjectMetadata(bucket string, fileName string, isDebug bool) (map[string]string, error) { headObjectInput := &s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } headObjectOutput, err := s.S3Client.HeadObject(context.TODO(), headObjectInput) if err != nil { return nil, err } return headObjectOutput.Metadata, nil } // MoveObjectBucketToBucket - Move object from one S3 bucket to another func (s ClientWithHelpers) MoveObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFileName string, settings S3UploadSettings) error { err := s.CopyObjectBucketToBucket(sourceBucket, destinationBucket, sourceFileName, destinationFileName, settings) if err != nil { return err } err = s.DeleteObjectFromBucket(sourceBucket, sourceFileName) if err != nil { return err } return nil } // CopyObjectBucketToBucket - Copy an object from one S3 bucket to another func (s ClientWithHelpers) CopyObjectBucketToBucket(sourceBucket string, destinationBucket string, sourceFileName string, destinationFilename string, settings S3UploadSettings) error { // copy the file copySource := url.QueryEscape(sourceBucket + "/" + sourceFileName) copyObjectInput := &s3.CopyObjectInput{ Bucket: aws.String(destinationBucket), // destination bucket CopySource: aws.String(copySource), // source path (ie: myBucket/myFile.csv) Key: aws.String(destinationFilename), // filename on destination } if settings.ExpiryDuration != nil { expiry := time.Now().Add(*settings.ExpiryDuration) copyObjectInput.Expires = &expiry } _, err := s.S3Client.CopyObject(context.TODO(), copyObjectInput) if err != nil { return err } // wait to see if the file copied successfully waiter := s3.NewObjectExistsWaiter(s.S3Client) err = waiter.Wait(context.TODO(), &s3.HeadObjectInput{Bucket: aws.String(destinationBucket), Key: aws.String(destinationFilename)}, 5*time.Minute) if err != nil { return err } return nil } // DeleteObjectFromBucket - Delete an object from an S3 bucket func (s ClientWithHelpers) DeleteObjectFromBucket(bucket string, fileName string) error { // delete the file deleteObjectInput := &s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(fileName), } _, err := s.S3Client.DeleteObject(context.TODO(), deleteObjectInput) if err != nil { return err } // wait to see if the file deleted successfully waiter := s3.NewObjectNotExistsWaiter(s.S3Client) err = waiter.Wait(context.TODO(), &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fileName)}, 5*time.Minute) if err != nil { return err } return nil } func GetS3FileKey(fileName string, folder string) string { var fileKey string fileName = strings.Trim(fileName, "/") folder = strings.Trim(folder, "/") if folder != "" { fileKey = fileName + "/" + folder } else { fileKey = fileName } return fileKey } func URLFromFileName(region string, bucket string, fileName string) string { logoUrl := "https://%s.s3.%s.amazonaws.com/%s" logoUrl = fmt.Sprintf(logoUrl, bucket, region, fileName) return logoUrl }