storage

README

Storage API

Deprecated - please use https://github.com/viant/afs API instead

This API provides unified way of accessing any storage system.

It comes with the following implementation so far:

<a name="import>

URL Scheme	Description	Pacakge
file	Local file system	github.com/viant/toolbox/storage
https	HTTP/s based system	github.com/viant/toolbox/storage
mem	Memory file system	github.com/viant/toolbox/storage
scp	SCP/SSH base systm	github.com/viant/toolbox/storage/scp
s3	Amazon Web Service S3	github.com/viant/toolbox/storage/aws
gs	Google Storage	github.com/viant/toolbox/storage/gs

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error

	//Closes storage service
	Close() error
}


//Object represents a storage object
type Object interface {
	//URL return storage url
	URL() string

	//Type returns storage type  StorageObjectFolderType or StorageObjectContentType
	Type() int

	//IsFolder returns true if object is a folder
	IsFolder() bool

	//IsContent returns true if object is a file
	IsContent() bool

	//Wrap wraps source storage object
	Wrap(source interface{})

	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error

	FileInfo() os.FileInfo
}

Usage:

    import (
    	"github.com/viant/toolbox/storage"
    	_ "github.com/viant/toolbox/storage/gs"
    	_ "github.com/viant/toolbox/storage/s3"
	
    )

    destinationURL := "gs://myBucket/set1/content.gz"
    destinationCredentialFile = "gs-secret.json"
	storageService, err := storage.NewServiceForURL(destinationURL, destinationCredentialFile)

    provider := storage.Registry().Get("s3")
    storageS3Service, err := provider("aws-secret.json")

Documentation

Overview

Package storage define abstract storage operation Deprecated - please use https://github.com/viant/afs API instead This package is frozen and no new functionality will be added, and future removal takes place.

Index

• Constants
• Variables
• func Archive(service Service, URL string, writer *zip.Writer) error
• func ArchiveWithFilter(service Service, URL string, writer *zip.Writer, ...) error
• func Copy(sourceService Service, sourceURL string, destinationService Service, ...) (err error)
• func Download(service Service, URL string) (io.ReadCloser, error)
• func DownloadText(service Service, URL string) (string, error)
• func GenerateStorageCode(mappings ...*StorageMapping) error
• func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, ...) os.FileInfo
• func NewFileMode(fileAttributes string) (os.FileMode, error)
• func Parse(URL string) (*url.URL, error)
• func Registry() *registry
• func ResetMemory()
• func Tar(service Service, URL string, writer *tar.Writer, includeOwnerDir bool) error
• type AbstractObject
  • func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject
  • func (o *AbstractObject) FileInfo() os.FileInfo
  • func (o *AbstractObject) IsContent() bool
  • func (o *AbstractObject) IsFolder() bool
  • func (o *AbstractObject) Type() int
  • func (o *AbstractObject) URL() string
  • func (o *AbstractObject) Wrap(source interface{})
• type CopyHandler
  • func NewStorageMapperHandler(filename, pkg string, useTextFormat bool, binaryFormat map[string]bool) (CopyHandler, io.WriteCloser, error)
• type MemoryFile
  • func (f *MemoryFile) Object() Object
• type MemoryFolder
  • func (f *MemoryFolder) Object() Object
  • func (f *MemoryFolder) Objects() []Object
• type ModificationHandler
• type Object
• type Provider
• type Service
  • func NewFileStorage() Service
  • func NewHttpStorageService(credential *cred.Config) Service
  • func NewMemoryService() Service
  • func NewPrivateMemoryService() Service
  • func NewService() Service
  • func NewServiceForURL(URL, credentials string) (Service, error)
• type StorageMapping

Constants

const (
	StorageObjectFolderType  int //folder type
	StorageObjectContentType     //file type
)

const FileProviderSchema = "file"

const HttpProviderScheme = "http"

const HttpsProviderScheme = "https"

const MemoryProviderScheme = "mem"

Variables

var DefaultFileMode os.FileMode = 0755

var HTTPClientProvider = func() (*http.Client, error) {
	return toolbox.NewHttpClient(&toolbox.HttpOptions{Key: "MaxIdleConns", Value: 0})
}

HTTPClientProvider represents http client provider

var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))

MemoryRoot represents memory root storage

Functions

func Archive

func Archive(service Service, URL string, writer *zip.Writer) error

Archive archives supplied URL assets into zip writer

func ArchiveWithFilter

func ArchiveWithFilter(service Service, URL string, writer *zip.Writer, predicate func(candidate Object) bool) error

Archive archives supplied URL assets into zip writer with supplied filter

func Copy

func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error)

Copy downloads objects from source URL to upload them to destination URL.

func Download

func Download(service Service, URL string) (io.ReadCloser, error)

Download returns a download reader for supplied URL

func DownloadText

func DownloadText(service Service, URL string) (string, error)

DownloadText returns a text for supplied URL

func GenerateStorageCode

func GenerateStorageCode(mappings ...*StorageMapping) error

GenerateStorageCode create a *.go files with statically scanned content from source URL.

func NewFileInfo

func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool) os.FileInfo

func NewFileMode

func NewFileMode(fileAttributes string) (os.FileMode, error)

func Parse

func Parse(URL string) (*url.URL, error)

func Registry

func Registry() *registry

Registry returns new provider

func ResetMemory

func ResetMemory()

ResetMemory reset memory root storage

func Tar

func Tar(service Service, URL string, writer *tar.Writer, includeOwnerDir bool) error

Tar tar archives supplied URL assets into zip writer

Types

type AbstractObject

type AbstractObject struct {
	Object

	Source interface{}
	// contains filtered or unexported fields
}

AbstractObject represents abstract storage object

func NewAbstractStorageObject

func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject

NewAbstractStorageObject creates a new abstract storage object

func (*AbstractObject) FileInfo

func (o *AbstractObject) FileInfo() os.FileInfo

func (*AbstractObject) IsContent

func (o *AbstractObject) IsContent() bool

IsContent returns true if object is a file

func (*AbstractObject) IsFolder

func (o *AbstractObject) IsFolder() bool

IsFolder returns true if object is a folder

func (*AbstractObject) Type

func (o *AbstractObject) Type() int

Type returns storage type StorageObjectFolderType or StorageObjectContentType

func (*AbstractObject) URL

func (o *AbstractObject) URL() string

URL return storage url

func (*AbstractObject) Wrap

func (o *AbstractObject) Wrap(source interface{})

Wrap wraps source storage object

type CopyHandler

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error

func NewStorageMapperHandler

func NewStorageMapperHandler(filename, pkg string, useTextFormat bool, binaryFormat map[string]bool) (CopyHandler, io.WriteCloser, error)

NewStorageMapperHandler creates a template handler for generating go file that write static content into memory service.

type MemoryFile

type MemoryFile struct {
	// contains filtered or unexported fields
}

func (*MemoryFile) Object

func (f *MemoryFile) Object() Object

type MemoryFolder

type MemoryFolder struct {
	// contains filtered or unexported fields
}

func (*MemoryFolder) Object

func (f *MemoryFolder) Object() Object

func (*MemoryFolder) Objects

func (f *MemoryFolder) Objects() []Object

type ModificationHandler

type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

type Object

type Object interface {
	//URL return storage url
	URL() string
	//Type returns storage type either folder or file
	Type() int
	//IsFolder returns true if object is a folder
	IsFolder() bool
	//IsContent returns true if object is a file
	IsContent() bool
	//Wrap wraps source storage object
	Wrap(source interface{})
	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error
	FileInfo() os.FileInfo
}

Object represents a storage object

type Provider

type Provider func(credentialFile string) (Service, error)

Provider represetns a service provider

type Service

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)
	//Exists returns true if resource exists
	Exists(URL string) (bool, error)
	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)
	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)
	//DownloadWithURL returns reader for downloaded URL object
	DownloadWithURL(URL string) (io.ReadCloser, error)
	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error
	//Upload uploads provided reader content for supplied storage object.
	UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error
	//Delete removes passed in storage object
	Delete(object Object) error
	//Register register schema with provided service
	Register(schema string, service Service) error
	//Closes storage service
	Close() error
}

Service represents abstract way to accessing local or remote storage

func NewFileStorage

func NewFileStorage() Service

func NewHttpStorageService

func NewHttpStorageService(credential *cred.Config) Service

func NewMemoryService

func NewMemoryService() Service

creates a new memory service

func NewPrivateMemoryService

func NewPrivateMemoryService() Service

creates a new private memory service

func NewService

func NewService() Service

NewService creates a new storage service

func NewServiceForURL

func NewServiceForURL(URL, credentials string) (Service, error)

NewServiceForURL creates a new storage service for provided URL scheme and optional credential file

type StorageMapping

type StorageMapping struct {
	SourceURL        string
	SourceCredential string
	DestinationURI   string
	TargetFile       string
	TargetPackage    string
	UseTextFormat    bool
}