s3zip

README

s3zip

s3zip zip files and upload to Amazon S3.

Motivation

Amazon S3 charges for Metadata and API requests. s3zip aims to reduce them.

For each object that is stored in the S3 Glacier Flexible Retrieval and S3 Glacier Deep Archive storage classes, AWS charges for 40 KB of additional metadata for each archived object, with 8 KB charged at S3 Standard rates and 32 KB charged at S3 Glacier Flexible Retrieval or S3 Deep Archive rates.

https://aws.amazon.com/s3/pricing

Usage

s3zip path/to/config.yaml

Config

s3:
  region: us-west-2
  bucket: my-bucket
  storage_class: DEEP_ARCHIVE # STANDARD | DEEP_ARCHIVE | etc.

targets:
  - path: D:\User\Desktop\MyPictures
    max_zip_depth: 2 # 0: zip MyPictures folder, 1: zip files under MyPictures/*, 2: zip files under MyPictures/**/*
    out_prefix: s3zip # prefix for s3 object key

Documentation

Index

• Constants
• Variables
• func Hash(name string) (string, error)
• func LocalObjects(path string, maxDepth int) ([]string, error)
• func Size(name string) (int, error)
• func Zip(name string) io.ReadCloser
• type Config
  • func ReadConfig(name string) (*Config, error)
• type ConfigS3
• type ConfigTarget
• type Metadata
  • func (*Metadata) Descriptor() ([]byte, []int)
  • func (x *Metadata) GetHash() string
  • func (*Metadata) ProtoMessage()
  • func (x *Metadata) ProtoReflect() protoreflect.Message
  • func (x *Metadata) Reset()
  • func (x *Metadata) String() string
• type MetadataStore
  • func (*MetadataStore) Descriptor() ([]byte, []int)
  • func (x *MetadataStore) GetMetadata() map[string]*Metadata
  • func (*MetadataStore) ProtoMessage()
  • func (x *MetadataStore) ProtoReflect() protoreflect.Message
  • func (x *MetadataStore) Reset()
  • func (x *MetadataStore) String() string
• type ObjectToUpload
• type RunInput
• type RunOutput
  • func Run(ctx context.Context, in *RunInput) (*RunOutput, error)

Constants

const (
	DefaultMetadataStoreKey = "s3zip-metadata.pb"
	DefaultConcurrency      = 10
)

Variables

var File_proto_metadata_proto protoreflect.FileDescriptor

Functions

func Hash

func Hash(name string) (string, error)

Hash returns a hash of the given file or directory. This only uses the file size and name for performance.

func LocalObjects

func LocalObjects(path string, maxDepth int) ([]string, error)

LocalObjects returns a list of relative paths to all files and directories. maxDepth is the maximum depth of recursion, 0 means no recursion.

func Size

func Size(name string) (int, error)

Size returns the size of the given file or directory.

func Zip

func Zip(name string) io.ReadCloser

Zip creates a zip file from the given directory.

Types

type Config

type Config struct {
	S3       ConfigS3       `yaml:"s3"`
	Metadata string         `yaml:"metadata"`
	Targets  []ConfigTarget `yaml:"targets"`
}

func ReadConfig

func ReadConfig(name string) (*Config, error)

type ConfigS3

type ConfigS3 struct {
	Region       string `yaml:"region"`
	Bucket       string `yaml:"bucket"`
	StorageClass string `yaml:"storage_class"`
}

type ConfigTarget

type ConfigTarget struct {
	Path        string `yaml:"path"`
	MaxZipDepth int    `yaml:"max_zip_depth"`
	OutPrefix   string `yaml:"out_prefix"`
}

type Metadata

type Metadata struct {
	Hash string `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
	// contains filtered or unexported fields
}

func (*Metadata) Descriptor

func (*Metadata) Descriptor() ([]byte, []int)

Deprecated: Use Metadata.ProtoReflect.Descriptor instead.

func (*Metadata) GetHash

func (x *Metadata) GetHash() string

func (*Metadata) ProtoMessage

func (*Metadata) ProtoMessage()

func (*Metadata) ProtoReflect

func (x *Metadata) ProtoReflect() protoreflect.Message

func (*Metadata) Reset

func (x *Metadata) Reset()

func (*Metadata) String

func (x *Metadata) String() string

type MetadataStore

type MetadataStore struct {
	Metadata map[string]*Metadata `` /* 143-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*MetadataStore) Descriptor

func (*MetadataStore) Descriptor() ([]byte, []int)

Deprecated: Use MetadataStore.ProtoReflect.Descriptor instead.

func (*MetadataStore) GetMetadata

func (x *MetadataStore) GetMetadata() map[string]*Metadata

func (*MetadataStore) ProtoMessage

func (*MetadataStore) ProtoMessage()

func (*MetadataStore) ProtoReflect

func (x *MetadataStore) ProtoReflect() protoreflect.Message

func (*MetadataStore) Reset

func (x *MetadataStore) Reset()

func (*MetadataStore) String

func (x *MetadataStore) String() string

type ObjectToUpload

type ObjectToUpload struct {
	Name string
	Hash string
	Size int
}

type RunInput

type RunInput struct {
	DryRun           bool
	S3Bucket         string
	S3Service        *s3.S3
	MetadataStoreKey string
	Path             string
	MaxZipDepth      int
	OutPrefix        string
	S3StorageClass   string
}

type RunOutput

type RunOutput struct {
	Upload int
	Delete int
}

func Run

func Run(ctx context.Context, in *RunInput) (*RunOutput, error)