backend

Documentation

Index

• type Backend
• type HdfsBackend
  • func NewHdfsBackend(client *hdfs.Client, namenode string, hdfsPath string) *HdfsBackend
  • func (h *HdfsBackend) DisplayPath(parts ...string) string
  • func (h *HdfsBackend) ListDBs() ([]string, error)
  • func (h *HdfsBackend) ListFiles(db, version string) ([]string, error)
  • func (h *HdfsBackend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)
  • func (h *HdfsBackend) Open(db, version, file string) (io.ReadCloser, error)
• type LocalBackend
  • func NewLocalBackend(path string) *LocalBackend
  • func (lb *LocalBackend) DisplayPath(parts ...string) string
  • func (lb *LocalBackend) ListDBs() ([]string, error)
  • func (lb *LocalBackend) ListFiles(db, version string) ([]string, error)
  • func (lb *LocalBackend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)
  • func (lb *LocalBackend) Open(db, version, file string) (io.ReadCloser, error)
• type S3Backend
  • func NewS3Backend(bucket string, s3path string, svc *s3.S3) *S3Backend
  • func (s *S3Backend) DisplayPath(parts ...string) string
  • func (s *S3Backend) ListDBs() ([]string, error)
  • func (s *S3Backend) ListFiles(db, version string) ([]string, error)
  • func (s *S3Backend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)
  • func (s *S3Backend) Open(db, version, file string) (io.ReadCloser, error)

Types

type Backend

type Backend interface {
	// Lists all valid DBs (the first set of directories under the source root).
	ListDBs() ([]string, error)

	// ListVersions returns a sorted list of valid versions for the given db. If
	// after is possed, only versions greater than it will be returned. If
	// checkForSuccess is passed, only versions for which there is a _SUCCESS file
	// present will be returned.
	ListVersions(db, after string, checkForSuccessFile bool) ([]string, error)

	// ListFiles returns a sorted list of all valid-looking data files for a db
	// and version. It excludes files that begin with '_' or '.'.
	ListFiles(db, version string) ([]string, error)

	// Open returns an io.ReadCloser for a given file from a specific version
	// of a db.
	Open(db, version, file string) (io.ReadCloser, error)

	// DisplayPath returns a human-readable path indicating where the backend
	// is rooted.
	DisplayPath(parts ...string) string
}

type HdfsBackend

type HdfsBackend struct {
	// contains filtered or unexported fields
}

func NewHdfsBackend

func NewHdfsBackend(client *hdfs.Client, namenode string, hdfsPath string) *HdfsBackend

func (*HdfsBackend) DisplayPath

func (h *HdfsBackend) DisplayPath(parts ...string) string

func (*HdfsBackend) ListDBs

func (h *HdfsBackend) ListDBs() ([]string, error)

func (*HdfsBackend) ListFiles

func (h *HdfsBackend) ListFiles(db, version string) ([]string, error)

func (*HdfsBackend) ListVersions

func (h *HdfsBackend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)

func (*HdfsBackend) Open

func (h *HdfsBackend) Open(db, version, file string) (io.ReadCloser, error)

type LocalBackend

type LocalBackend struct {
	// contains filtered or unexported fields
}

A basic backend for the local filesystem

func NewLocalBackend

func NewLocalBackend(path string) *LocalBackend

func (*LocalBackend) DisplayPath

func (lb *LocalBackend) DisplayPath(parts ...string) string

func (*LocalBackend) ListDBs

func (lb *LocalBackend) ListDBs() ([]string, error)

func (*LocalBackend) ListFiles

func (lb *LocalBackend) ListFiles(db, version string) ([]string, error)

func (*LocalBackend) ListVersions

func (lb *LocalBackend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)

func (*LocalBackend) Open

func (lb *LocalBackend) Open(db, version, file string) (io.ReadCloser, error)

type S3Backend

type S3Backend struct {
	// contains filtered or unexported fields
}

func NewS3Backend

func NewS3Backend(bucket string, s3path string, svc *s3.S3) *S3Backend

func (*S3Backend) DisplayPath

func (s *S3Backend) DisplayPath(parts ...string) string

func (*S3Backend) ListDBs

func (s *S3Backend) ListDBs() ([]string, error)

func (*S3Backend) ListFiles

func (s *S3Backend) ListFiles(db, version string) ([]string, error)

func (*S3Backend) ListVersions

func (s *S3Backend) ListVersions(db, after string, checkForSuccess bool) ([]string, error)

func (*S3Backend) Open

func (s *S3Backend) Open(db, version, file string) (io.ReadCloser, error)