Documentation

Overview

    Package s3manager provides utilities to upload and download objects from S3 concurrently. Helpful for when working with large objects.

    Index

    Constants

    View Source
    const (
    	// DefaultBatchSize is the batch size we initialize when constructing a batch delete client.
    	// This value is used when calling DeleteObjects. This represents how many objects to delete
    	// per DeleteObjects call.
    	DefaultBatchSize = 100
    )
    View Source
    const DefaultDownloadConcurrency = 5

      DefaultDownloadConcurrency is the default number of goroutines to spin up when using Download().

      View Source
      const DefaultDownloadPartSize = 1024 * 1024 * 5

        DefaultDownloadPartSize is the default range of bytes to get at a time when using Download().

        View Source
        const DefaultUploadConcurrency = 5

          DefaultUploadConcurrency is the default number of goroutines to spin up when using Upload().

          View Source
          const DefaultUploadPartSize = MinUploadPartSize

            DefaultUploadPartSize is the default part size to buffer chunks of a payload into.

            View Source
            const MaxUploadParts = 10000

              MaxUploadParts is the maximum allowed number of parts in a multi-part upload on Amazon S3.

              View Source
              const MinUploadPartSize int64 = 1024 * 1024 * 5

                MinUploadPartSize is the minimum allowed part size when uploading a part to Amazon S3.

                Variables

                This section is empty.

                Functions

                func GetBucketRegion

                func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error)

                  GetBucketRegion will attempt to get the region for a bucket using the regionHint to determine which AWS partition to perform the query on.

                  The request will not be signed, and will not use your AWS credentials.

                  A "NotFound" error code will be returned if the bucket does not exist in the AWS partition the regionHint belongs to.

                  For example to get the region of a bucket which exists in "eu-central-1" you could provide a region hint of "us-west-2".

                  sess := session.Must(session.NewSession())
                  
                  bucket := "my-bucket"
                  region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2")
                  if err != nil {
                      if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" {
                           fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket)
                      }
                      return err
                  }
                  fmt.Printf("Bucket %s is in %s region\n", bucket, region)
                  

                  func GetBucketRegionWithClient

                  func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error)

                    GetBucketRegionWithClient is the same as GetBucketRegion with the exception that it takes a S3 service client instead of a Session. The regionHint is derived from the region the S3 service client was created in.

                    See GetBucketRegion for more information.

                    func NewBatchError

                    func NewBatchError(code, message string, err []Error) awserr.Error

                      NewBatchError will return a BatchError that satisfies the awserr.Error interface.

                      func WithDownloaderRequestOptions

                      func WithDownloaderRequestOptions(opts ...request.Option) func(*Downloader)

                        WithDownloaderRequestOptions appends to the Downloader's API request options.

                        func WithUploaderRequestOptions

                        func WithUploaderRequestOptions(opts ...request.Option) func(*Uploader)

                          WithUploaderRequestOptions appends to the Uploader's API request options.

                          Types

                          type BatchDelete

                          type BatchDelete struct {
                          	Client    s3iface.S3API
                          	BatchSize int
                          }

                            BatchDelete will use the s3 package's service client to perform a batch delete.

                            func NewBatchDelete

                            func NewBatchDelete(c client.ConfigProvider, options ...func(*BatchDelete)) *BatchDelete

                              NewBatchDelete will return a new delete client that can delete a batched amount of objects.

                              Example:

                              batcher := s3manager.NewBatchDelete(sess, size)
                              
                              objects := []BatchDeleteObject{
                              	{
                              		Object:	&s3.DeleteObjectInput {
                              			Key: aws.String("key"),
                              			Bucket: aws.String("bucket"),
                              		},
                              	},
                              }
                              
                              if err := batcher.Delete(&s3manager.DeleteObjectsIterator{
                              	Objects: objects,
                              }); err != nil {
                              	return err
                              }
                              

                              func NewBatchDeleteWithClient

                              func NewBatchDeleteWithClient(client s3iface.S3API, options ...func(*BatchDelete)) *BatchDelete

                                NewBatchDeleteWithClient will return a new delete client that can delete a batched amount of objects.

                                Example:

                                batcher := s3manager.NewBatchDeleteWithClient(client, size)
                                
                                objects := []BatchDeleteObject{
                                	{
                                		Object:	&s3.DeleteObjectInput {
                                			Key: aws.String("key"),
                                			Bucket: aws.String("bucket"),
                                		},
                                	},
                                }
                                
                                if err := batcher.Delete(&s3manager.DeleteObjectsIterator{
                                	Objects: objects,
                                }); err != nil {
                                	return err
                                }
                                

                                func (*BatchDelete) Delete

                                func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator) error

                                  Delete will use the iterator to queue up objects that need to be deleted. Once the batch size is met, this will call the deleteBatch function.

                                  type BatchDeleteIterator

                                  type BatchDeleteIterator interface {
                                  	Next() bool
                                  	Err() error
                                  	DeleteObject() BatchDeleteObject
                                  }

                                    BatchDeleteIterator is an interface that uses the scanner pattern to iterate through what needs to be deleted.

                                    func NewDeleteListIterator

                                    func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts ...func(*DeleteListIterator)) BatchDeleteIterator

                                      NewDeleteListIterator will return a new DeleteListIterator.

                                      type BatchDeleteObject

                                      type BatchDeleteObject struct {
                                      	Object *s3.DeleteObjectInput
                                      	// After will run after each iteration during the batch process. This function will
                                      	// be executed whether or not the request was successful.
                                      	After func() error
                                      }

                                        BatchDeleteObject is a wrapper object for calling the batch delete operation.

                                        type BatchDownloadIterator

                                        type BatchDownloadIterator interface {
                                        	Next() bool
                                        	Err() error
                                        	DownloadObject() BatchDownloadObject
                                        }

                                          BatchDownloadIterator is an interface that uses the scanner pattern to iterate through a series of objects to be downloaded.

                                          type BatchDownloadObject

                                          type BatchDownloadObject struct {
                                          	Object *s3.GetObjectInput
                                          	Writer io.WriterAt
                                          	// After will run after each iteration during the batch process. This function will
                                          	// be executed whether or not the request was successful.
                                          	After func() error
                                          }

                                            BatchDownloadObject contains all necessary information to run a batch operation once.

                                            type BatchError

                                            type BatchError struct {
                                            	Errors Errors
                                            	// contains filtered or unexported fields
                                            }

                                              BatchError will contain the key and bucket of the object that failed to either upload or download.

                                              func (*BatchError) Code

                                              func (err *BatchError) Code() string

                                                Code will return the code associated with the batch error.

                                                func (*BatchError) Error

                                                func (err *BatchError) Error() string

                                                func (*BatchError) Message

                                                func (err *BatchError) Message() string

                                                  Message will return the message associated with the batch error.

                                                  func (*BatchError) OrigErr

                                                  func (err *BatchError) OrigErr() error

                                                    OrigErr will return the original error. Which, in this case, will always be nil for batched operations.

                                                    type BatchUploadIterator

                                                    type BatchUploadIterator interface {
                                                    	Next() bool
                                                    	Err() error
                                                    	UploadObject() BatchUploadObject
                                                    }

                                                      BatchUploadIterator is an interface that uses the scanner pattern to iterate through what needs to be uploaded.

                                                      type BatchUploadObject

                                                      type BatchUploadObject struct {
                                                      	Object *UploadInput
                                                      	// After will run after each iteration during the batch process. This function will
                                                      	// be executed whether or not the request was successful.
                                                      	After func() error
                                                      }

                                                        BatchUploadObject contains all necessary information to run a batch operation once.

                                                        type DeleteListIterator

                                                        type DeleteListIterator struct {
                                                        	Bucket    *string
                                                        	Paginator request.Pagination
                                                        	// contains filtered or unexported fields
                                                        }

                                                          DeleteListIterator is an alternative iterator for the BatchDelete client. This will iterate through a list of objects and delete the objects.

                                                          Example:

                                                          iter := &s3manager.DeleteListIterator{
                                                          	Client: svc,
                                                          	Input: &s3.ListObjectsInput{
                                                          		Bucket:  aws.String("bucket"),
                                                          		MaxKeys: aws.Int64(5),
                                                          	},
                                                          	Paginator: request.Pagination{
                                                          		NewRequest: func() (*request.Request, error) {
                                                          			var inCpy *ListObjectsInput
                                                          			if input != nil {
                                                          				tmp := *input
                                                          				inCpy = &tmp
                                                          			}
                                                          			req, _ := c.ListObjectsRequest(inCpy)
                                                          			return req, nil
                                                          		},
                                                          	},
                                                          }
                                                          
                                                          batcher := s3manager.NewBatchDeleteWithClient(svc)
                                                          if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil {
                                                          	return err
                                                          }
                                                          

                                                          func (*DeleteListIterator) DeleteObject

                                                          func (iter *DeleteListIterator) DeleteObject() BatchDeleteObject

                                                            DeleteObject will return the current object to be deleted.

                                                            func (*DeleteListIterator) Err

                                                            func (iter *DeleteListIterator) Err() error

                                                              Err will return the last known error from Next.

                                                              func (*DeleteListIterator) Next

                                                              func (iter *DeleteListIterator) Next() bool

                                                                Next will use the S3API client to iterate through a list of objects.

                                                                type DeleteObjectsIterator

                                                                type DeleteObjectsIterator struct {
                                                                	Objects []BatchDeleteObject
                                                                	// contains filtered or unexported fields
                                                                }

                                                                  DeleteObjectsIterator is an interface that uses the scanner pattern to iterate through a series of objects to be deleted.

                                                                  func (*DeleteObjectsIterator) DeleteObject

                                                                  func (iter *DeleteObjectsIterator) DeleteObject() BatchDeleteObject

                                                                    DeleteObject will return the BatchDeleteObject at the current batched index.

                                                                    func (*DeleteObjectsIterator) Err

                                                                    func (iter *DeleteObjectsIterator) Err() error

                                                                      Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface this will only return nil.

                                                                      func (*DeleteObjectsIterator) Next

                                                                      func (iter *DeleteObjectsIterator) Next() bool

                                                                        Next will increment the default iterator's index and and ensure that there is another object to iterator to.

                                                                        type DownloadObjectsIterator

                                                                        type DownloadObjectsIterator struct {
                                                                        	Objects []BatchDownloadObject
                                                                        	// contains filtered or unexported fields
                                                                        }

                                                                          DownloadObjectsIterator implements the BatchDownloadIterator interface and allows for batched download of objects.

                                                                          func (*DownloadObjectsIterator) DownloadObject

                                                                          func (batcher *DownloadObjectsIterator) DownloadObject() BatchDownloadObject

                                                                            DownloadObject will return the BatchDownloadObject at the current batched index.

                                                                            func (*DownloadObjectsIterator) Err

                                                                            func (batcher *DownloadObjectsIterator) Err() error

                                                                              Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface this will only return nil.

                                                                              func (*DownloadObjectsIterator) Next

                                                                              func (batcher *DownloadObjectsIterator) Next() bool

                                                                                Next will increment the default iterator's index and and ensure that there is another object to iterator to.

                                                                                type Downloader

                                                                                type Downloader struct {
                                                                                	// The buffer size (in bytes) to use when buffering data into chunks and
                                                                                	// sending them as parts to S3. The minimum allowed part size is 5MB, and
                                                                                	// if this value is set to zero, the DefaultDownloadPartSize value will be used.
                                                                                	//
                                                                                	// PartSize is ignored if the Range input parameter is provided.
                                                                                	PartSize int64
                                                                                
                                                                                	// The number of goroutines to spin up in parallel when sending parts.
                                                                                	// If this is set to zero, the DefaultDownloadConcurrency value will be used.
                                                                                	//
                                                                                	// Concurrency is ignored if the Range input parameter is provided.
                                                                                	Concurrency int
                                                                                
                                                                                	// An S3 client to use when performing downloads.
                                                                                	S3 s3iface.S3API
                                                                                
                                                                                	// List of request options that will be passed down to individual API
                                                                                	// operation requests made by the downloader.
                                                                                	RequestOptions []request.Option
                                                                                }

                                                                                  The Downloader structure that calls Download(). It is safe to call Download() on this structure for multiple objects and across concurrent goroutines. Mutating the Downloader's properties is not safe to be done concurrently.

                                                                                  func NewDownloader

                                                                                  func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downloader

                                                                                    NewDownloader creates a new Downloader instance to downloads objects from S3 in concurrent chunks. Pass in additional functional options to customize the downloader behavior. Requires a client.ConfigProvider in order to create a S3 service client. The session.Session satisfies the client.ConfigProvider interface.

                                                                                    Example:

                                                                                    // The session the S3 Downloader will use
                                                                                    sess := session.Must(session.NewSession())
                                                                                    
                                                                                    // Create a downloader with the session and default options
                                                                                    downloader := s3manager.NewDownloader(sess)
                                                                                    
                                                                                    // Create a downloader with the session and custom options
                                                                                    downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
                                                                                         d.PartSize = 64 * 1024 * 1024 // 64MB per part
                                                                                    })
                                                                                    

                                                                                    func NewDownloaderWithClient

                                                                                    func NewDownloaderWithClient(svc s3iface.S3API, options ...func(*Downloader)) *Downloader

                                                                                      NewDownloaderWithClient creates a new Downloader instance to downloads objects from S3 in concurrent chunks. Pass in additional functional options to customize the downloader behavior. Requires a S3 service client to make S3 API calls.

                                                                                      Example:

                                                                                      // The session the S3 Downloader will use
                                                                                      sess := session.Must(session.NewSession())
                                                                                      
                                                                                      // The S3 client the S3 Downloader will use
                                                                                      s3Svc := s3.new(sess)
                                                                                      
                                                                                      // Create a downloader with the s3 client and default options
                                                                                      downloader := s3manager.NewDownloaderWithClient(s3Svc)
                                                                                      
                                                                                      // Create a downloader with the s3 client and custom options
                                                                                      downloader := s3manager.NewDownloaderWithClient(s3Svc, func(d *s3manager.Downloader) {
                                                                                           d.PartSize = 64 * 1024 * 1024 // 64MB per part
                                                                                      })
                                                                                      

                                                                                      func (Downloader) Download

                                                                                      func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error)

                                                                                        Download downloads an object in S3 and writes the payload into w using concurrent GET requests.

                                                                                        Additional functional options can be provided to configure the individual download. These options are copies of the Downloader instance Download is called from. Modifying the options will not impact the original Downloader instance.

                                                                                        It is safe to call this method concurrently across goroutines.

                                                                                        The w io.WriterAt can be satisfied by an os.File to do multipart concurrent downloads, or in memory []byte wrapper using aws.WriteAtBuffer.

                                                                                        If the GetObjectInput's Range value is provided that will cause the downloader to perform a single GetObjectInput request for that object's range. This will caused the part size, and concurrency configurations to be ignored.

                                                                                        func (Downloader) DownloadWithContext

                                                                                        func (d Downloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error)

                                                                                          DownloadWithContext downloads an object in S3 and writes the payload into w using concurrent GET requests.

                                                                                          DownloadWithContext is the same as Download with the additional support for Context input parameters. The Context must not be nil. A nil Context will cause a panic. Use the Context to add deadlining, timeouts, ect. The DownloadWithContext may create sub-contexts for individual underlying requests.

                                                                                          Additional functional options can be provided to configure the individual download. These options are copies of the Downloader instance Download is called from. Modifying the options will not impact the original Downloader instance. Use the WithDownloaderRequestOptions helper function to pass in request options that will be applied to all API operations made with this downloader.

                                                                                          The w io.WriterAt can be satisfied by an os.File to do multipart concurrent downloads, or in memory []byte wrapper using aws.WriteAtBuffer.

                                                                                          It is safe to call this method concurrently across goroutines.

                                                                                          If the GetObjectInput's Range value is provided that will cause the downloader to perform a single GetObjectInput request for that object's range. This will caused the part size, and concurrency configurations to be ignored.

                                                                                          func (Downloader) DownloadWithIterator

                                                                                          func (d Downloader) DownloadWithIterator(ctx aws.Context, iter BatchDownloadIterator, opts ...func(*Downloader)) error

                                                                                            DownloadWithIterator will download a batched amount of objects in S3 and writes them to the io.WriterAt specificed in the iterator.

                                                                                            Example:

                                                                                            svc := s3manager.NewDownloader(session)
                                                                                            
                                                                                            fooFile, err := os.Open("/tmp/foo.file")
                                                                                            if err != nil {
                                                                                            	return err
                                                                                            }
                                                                                            
                                                                                            barFile, err := os.Open("/tmp/bar.file")
                                                                                            if err != nil {
                                                                                            	return err
                                                                                            }
                                                                                            
                                                                                            objects := []s3manager.BatchDownloadObject {
                                                                                            	{
                                                                                            		Input: &s3.GetObjectInput {
                                                                                            			Bucket: aws.String("bucket"),
                                                                                            			Key: aws.String("foo"),
                                                                                            		},
                                                                                            		Writer: fooFile,
                                                                                            	},
                                                                                            	{
                                                                                            		Input: &s3.GetObjectInput {
                                                                                            			Bucket: aws.String("bucket"),
                                                                                            			Key: aws.String("bar"),
                                                                                            		},
                                                                                            		Writer: barFile,
                                                                                            	},
                                                                                            }
                                                                                            
                                                                                            iter := &s3manager.DownloadObjectsIterator{Objects: objects}
                                                                                            if err := svc.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil {
                                                                                            	return err
                                                                                            }
                                                                                            

                                                                                            type Error

                                                                                            type Error struct {
                                                                                            	OrigErr error
                                                                                            	Bucket  *string
                                                                                            	Key     *string
                                                                                            }

                                                                                              Error will contain the original error, bucket, and key of the operation that failed during batch operations.

                                                                                              func (*Error) Error

                                                                                              func (err *Error) Error() string

                                                                                              type Errors

                                                                                              type Errors []Error

                                                                                                Errors is a typed alias for a slice of errors to satisfy the error interface.

                                                                                                func (Errors) Error

                                                                                                func (errs Errors) Error() string

                                                                                                type MultiUploadFailure

                                                                                                type MultiUploadFailure interface {
                                                                                                	awserr.Error
                                                                                                
                                                                                                	// Returns the upload id for the S3 multipart upload that failed.
                                                                                                	UploadID() string
                                                                                                }

                                                                                                  A MultiUploadFailure wraps a failed S3 multipart upload. An error returned will satisfy this interface when a multi part upload failed to upload all chucks to S3. In the case of a failure the UploadID is needed to operate on the chunks, if any, which were uploaded.

                                                                                                  Example:

                                                                                                  u := s3manager.NewUploader(opts)
                                                                                                  output, err := u.upload(input)
                                                                                                  if err != nil {
                                                                                                      if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
                                                                                                          // Process error and its associated uploadID
                                                                                                          fmt.Println("Error:", multierr.Code(), multierr.Message(), multierr.UploadID())
                                                                                                      } else {
                                                                                                          // Process error generically
                                                                                                          fmt.Println("Error:", err.Error())
                                                                                                      }
                                                                                                  }
                                                                                                  

                                                                                                  type UploadInput

                                                                                                  type UploadInput struct {
                                                                                                  	// The canned ACL to apply to the object.
                                                                                                  	ACL *string `location:"header" locationName:"x-amz-acl" type:"string"`
                                                                                                  
                                                                                                  	Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
                                                                                                  
                                                                                                  	// Specifies caching behavior along the request/reply chain.
                                                                                                  	CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"`
                                                                                                  
                                                                                                  	// Specifies presentational information for the object.
                                                                                                  	ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"`
                                                                                                  
                                                                                                  	// Specifies what content encodings have been applied to the object and thus
                                                                                                  	// what decoding mechanisms must be applied to obtain the media-type referenced
                                                                                                  	// by the Content-Type header field.
                                                                                                  	ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"`
                                                                                                  
                                                                                                  	// The language the content is in.
                                                                                                  	ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"`
                                                                                                  
                                                                                                  	// The base64-encoded 128-bit MD5 digest of the part data.
                                                                                                  	ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"`
                                                                                                  
                                                                                                  	// A standard MIME type describing the format of the object data.
                                                                                                  	ContentType *string `location:"header" locationName:"Content-Type" type:"string"`
                                                                                                  
                                                                                                  	// The date and time at which the object is no longer cacheable.
                                                                                                  	Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"`
                                                                                                  
                                                                                                  	// Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
                                                                                                  	GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"`
                                                                                                  
                                                                                                  	// Allows grantee to read the object data and its metadata.
                                                                                                  	GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"`
                                                                                                  
                                                                                                  	// Allows grantee to read the object ACL.
                                                                                                  	GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"`
                                                                                                  
                                                                                                  	// Allows grantee to write the ACL for the applicable object.
                                                                                                  	GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"`
                                                                                                  
                                                                                                  	Key *string `location:"uri" locationName:"Key" type:"string" required:"true"`
                                                                                                  
                                                                                                  	// A map of metadata to store with the object in S3.
                                                                                                  	Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"`
                                                                                                  
                                                                                                  	// Confirms that the requester knows that she or he will be charged for the
                                                                                                  	// request. Bucket owners need not specify this parameter in their requests.
                                                                                                  	// Documentation on downloading objects from requester pays buckets can be found
                                                                                                  	// at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
                                                                                                  	RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"`
                                                                                                  
                                                                                                  	// Specifies the algorithm to use to when encrypting the object (e.g., AES256,
                                                                                                  	// aws:kms).
                                                                                                  	SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"`
                                                                                                  
                                                                                                  	// Specifies the customer-provided encryption key for Amazon S3 to use in encrypting
                                                                                                  	// data. This value is used to store the object and then it is discarded; Amazon
                                                                                                  	// does not store the encryption key. The key must be appropriate for use with
                                                                                                  	// the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm
                                                                                                  	// header.
                                                                                                  	SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"`
                                                                                                  
                                                                                                  	// Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
                                                                                                  	// Amazon S3 uses this header for a message integrity check to ensure the encryption
                                                                                                  	// key was transmitted without error.
                                                                                                  	SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
                                                                                                  
                                                                                                  	// Specifies the AWS KMS key ID to use for object encryption. All GET and PUT
                                                                                                  	// requests for an object protected by AWS KMS will fail if not made via SSL
                                                                                                  	// or using SigV4. Documentation on configuring any of the officially supported
                                                                                                  	// AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version
                                                                                                  	SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"`
                                                                                                  
                                                                                                  	// The Server-side encryption algorithm used when storing this object in S3
                                                                                                  	// (e.g., AES256, aws:kms).
                                                                                                  	ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"`
                                                                                                  
                                                                                                  	// The type of storage to use for the object. Defaults to 'STANDARD'.
                                                                                                  	StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"`
                                                                                                  
                                                                                                  	// The tag-set for the object. The tag-set must be encoded as URL Query parameters
                                                                                                  	Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"`
                                                                                                  
                                                                                                  	// If the bucket is configured as a website, redirects requests for this object
                                                                                                  	// to another object in the same bucket or to an external URL. Amazon S3 stores
                                                                                                  	// the value of this header in the object metadata.
                                                                                                  	WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"`
                                                                                                  
                                                                                                  	// The readable body payload to send to S3.
                                                                                                  	Body io.Reader
                                                                                                  }

                                                                                                    UploadInput contains all input for upload requests to Amazon S3.

                                                                                                    type UploadObjectsIterator

                                                                                                    type UploadObjectsIterator struct {
                                                                                                    	Objects []BatchUploadObject
                                                                                                    	// contains filtered or unexported fields
                                                                                                    }

                                                                                                      UploadObjectsIterator implements the BatchUploadIterator interface and allows for batched upload of objects.

                                                                                                      func (*UploadObjectsIterator) Err

                                                                                                      func (batcher *UploadObjectsIterator) Err() error

                                                                                                        Err will return an error. Since this is just used to satisfy the BatchUploadIterator interface this will only return nil.

                                                                                                        func (*UploadObjectsIterator) Next

                                                                                                        func (batcher *UploadObjectsIterator) Next() bool

                                                                                                          Next will increment the default iterator's index and and ensure that there is another object to iterator to.

                                                                                                          func (*UploadObjectsIterator) UploadObject

                                                                                                          func (batcher *UploadObjectsIterator) UploadObject() BatchUploadObject

                                                                                                            UploadObject will return the BatchUploadObject at the current batched index.

                                                                                                            type UploadOutput

                                                                                                            type UploadOutput struct {
                                                                                                            	// The URL where the object was uploaded to.
                                                                                                            	Location string
                                                                                                            
                                                                                                            	// The version of the object that was uploaded. Will only be populated if
                                                                                                            	// the S3 Bucket is versioned. If the bucket is not versioned this field
                                                                                                            	// will not be set.
                                                                                                            	VersionID *string
                                                                                                            
                                                                                                            	// The ID for a multipart upload to S3. In the case of an error the error
                                                                                                            	// can be cast to the MultiUploadFailure interface to extract the upload ID.
                                                                                                            	UploadID string
                                                                                                            }

                                                                                                              UploadOutput represents a response from the Upload() call.

                                                                                                              type Uploader

                                                                                                              type Uploader struct {
                                                                                                              	// The buffer size (in bytes) to use when buffering data into chunks and
                                                                                                              	// sending them as parts to S3. The minimum allowed part size is 5MB, and
                                                                                                              	// if this value is set to zero, the DefaultUploadPartSize value will be used.
                                                                                                              	PartSize int64
                                                                                                              
                                                                                                              	// The number of goroutines to spin up in parallel per call to Upload when
                                                                                                              	// sending parts. If this is set to zero, the DefaultUploadConcurrency value
                                                                                                              	// will be used.
                                                                                                              	//
                                                                                                              	// The concurrency pool is not shared between calls to Upload.
                                                                                                              	Concurrency int
                                                                                                              
                                                                                                              	// Setting this value to true will cause the SDK to avoid calling
                                                                                                              	// AbortMultipartUpload on a failure, leaving all successfully uploaded
                                                                                                              	// parts on S3 for manual recovery.
                                                                                                              	//
                                                                                                              	// Note that storing parts of an incomplete multipart upload counts towards
                                                                                                              	// space usage on S3 and will add additional costs if not cleaned up.
                                                                                                              	LeavePartsOnError bool
                                                                                                              
                                                                                                              	// MaxUploadParts is the max number of parts which will be uploaded to S3.
                                                                                                              	// Will be used to calculate the partsize of the object to be uploaded.
                                                                                                              	// E.g: 5GB file, with MaxUploadParts set to 100, will upload the file
                                                                                                              	// as 100, 50MB parts.
                                                                                                              	// With a limited of s3.MaxUploadParts (10,000 parts).
                                                                                                              	MaxUploadParts int
                                                                                                              
                                                                                                              	// The client to use when uploading to S3.
                                                                                                              	S3 s3iface.S3API
                                                                                                              
                                                                                                              	// List of request options that will be passed down to individual API
                                                                                                              	// operation requests made by the uploader.
                                                                                                              	RequestOptions []request.Option
                                                                                                              }

                                                                                                                The Uploader structure that calls Upload(). It is safe to call Upload() on this structure for multiple objects and across concurrent goroutines. Mutating the Uploader's properties is not safe to be done concurrently.

                                                                                                                func NewUploader

                                                                                                                func NewUploader(c client.ConfigProvider, options ...func(*Uploader)) *Uploader

                                                                                                                  NewUploader creates a new Uploader instance to upload objects to S3. Pass In additional functional options to customize the uploader's behavior. Requires a client.ConfigProvider in order to create a S3 service client. The session.Session satisfies the client.ConfigProvider interface.

                                                                                                                  Example:

                                                                                                                  // The session the S3 Uploader will use
                                                                                                                  sess := session.Must(session.NewSession())
                                                                                                                  
                                                                                                                  // Create an uploader with the session and default options
                                                                                                                  uploader := s3manager.NewUploader(sess)
                                                                                                                  
                                                                                                                  // Create an uploader with the session and custom options
                                                                                                                  uploader := s3manager.NewUploader(session, func(u *s3manager.Uploader) {
                                                                                                                       u.PartSize = 64 * 1024 * 1024 // 64MB per part
                                                                                                                  })
                                                                                                                  

                                                                                                                  func NewUploaderWithClient

                                                                                                                  func NewUploaderWithClient(svc s3iface.S3API, options ...func(*Uploader)) *Uploader

                                                                                                                    NewUploaderWithClient creates a new Uploader instance to upload objects to S3. Pass in additional functional options to customize the uploader's behavior. Requires a S3 service client to make S3 API calls.

                                                                                                                    Example:

                                                                                                                    // The session the S3 Uploader will use
                                                                                                                    sess := session.Must(session.NewSession())
                                                                                                                    
                                                                                                                    // S3 service client the Upload manager will use.
                                                                                                                    s3Svc := s3.New(sess)
                                                                                                                    
                                                                                                                    // Create an uploader with S3 client and default options
                                                                                                                    uploader := s3manager.NewUploaderWithClient(s3Svc)
                                                                                                                    
                                                                                                                    // Create an uploader with S3 client and custom options
                                                                                                                    uploader := s3manager.NewUploaderWithClient(s3Svc, func(u *s3manager.Uploader) {
                                                                                                                         u.PartSize = 64 * 1024 * 1024 // 64MB per part
                                                                                                                    })
                                                                                                                    

                                                                                                                    func (Uploader) Upload

                                                                                                                    func (u Uploader) Upload(input *UploadInput, options ...func(*Uploader)) (*UploadOutput, error)

                                                                                                                      Upload uploads an object to S3, intelligently buffering large files into smaller chunks and sending them in parallel across multiple goroutines. You can configure the buffer size and concurrency through the Uploader's parameters.

                                                                                                                      Additional functional options can be provided to configure the individual upload. These options are copies of the Uploader instance Upload is called from. Modifying the options will not impact the original Uploader instance.

                                                                                                                      Use the WithUploaderRequestOptions helper function to pass in request options that will be applied to all API operations made with this uploader.

                                                                                                                      It is safe to call this method concurrently across goroutines.

                                                                                                                      Example:

                                                                                                                      // Upload input parameters
                                                                                                                      upParams := &s3manager.UploadInput{
                                                                                                                          Bucket: &bucketName,
                                                                                                                          Key:    &keyName,
                                                                                                                          Body:   file,
                                                                                                                      }
                                                                                                                      
                                                                                                                      // Perform an upload.
                                                                                                                      result, err := uploader.Upload(upParams)
                                                                                                                      
                                                                                                                      // Perform upload with options different than the those in the Uploader.
                                                                                                                      result, err := uploader.Upload(upParams, func(u *s3manager.Uploader) {
                                                                                                                           u.PartSize = 10 * 1024 * 1024 // 10MB part size
                                                                                                                           u.LeavePartsOnError = true    // Don't delete the parts if the upload fails.
                                                                                                                      })
                                                                                                                      

                                                                                                                      func (Uploader) UploadWithContext

                                                                                                                      func (u Uploader) UploadWithContext(ctx aws.Context, input *UploadInput, opts ...func(*Uploader)) (*UploadOutput, error)

                                                                                                                        UploadWithContext uploads an object to S3, intelligently buffering large files into smaller chunks and sending them in parallel across multiple goroutines. You can configure the buffer size and concurrency through the Uploader's parameters.

                                                                                                                        UploadWithContext is the same as Upload with the additional support for Context input parameters. The Context must not be nil. A nil Context will cause a panic. Use the context to add deadlining, timeouts, ect. The UploadWithContext may create sub-contexts for individual underlying requests.

                                                                                                                        Additional functional options can be provided to configure the individual upload. These options are copies of the Uploader instance Upload is called from. Modifying the options will not impact the original Uploader instance.

                                                                                                                        Use the WithUploaderRequestOptions helper function to pass in request options that will be applied to all API operations made with this uploader.

                                                                                                                        It is safe to call this method concurrently across goroutines.

                                                                                                                        func (Uploader) UploadWithIterator

                                                                                                                        func (u Uploader) UploadWithIterator(ctx aws.Context, iter BatchUploadIterator, opts ...func(*Uploader)) error

                                                                                                                          UploadWithIterator will upload a batched amount of objects to S3. This operation uses the iterator pattern to know which object to upload next. Since this is an interface this allows for custom defined functionality.

                                                                                                                          Example:

                                                                                                                          svc:= s3manager.NewUploader(sess)
                                                                                                                          
                                                                                                                          objects := []BatchUploadObject{
                                                                                                                          	{
                                                                                                                          		Object:	&s3manager.UploadInput {
                                                                                                                          			Key: aws.String("key"),
                                                                                                                          			Bucket: aws.String("bucket"),
                                                                                                                          		},
                                                                                                                          	},
                                                                                                                          }
                                                                                                                          
                                                                                                                          iter := &s3managee.UploadObjectsIterator{Objects: objects}
                                                                                                                          if err := svc.UploadWithIterator(aws.BackgroundContext(), iter); err != nil {
                                                                                                                          	return err
                                                                                                                          }
                                                                                                                          

                                                                                                                          Directories

                                                                                                                          Path Synopsis
                                                                                                                          Package s3manageriface provides an interface for the s3manager package
                                                                                                                          Package s3manageriface provides an interface for the s3manager package