hdfs

Documentation

Overview

Package hdfs provides an interface to the HDFS storage system.

Index

• func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error)
• type Fs
  • func (f *Fs) About(ctx context.Context) (*fs.Usage, error)
  • func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error)
  • func (f *Fs) Features() *fs.Features
  • func (f *Fs) Hashes() hash.Set
  • func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error)
  • func (f *Fs) Mkdir(ctx context.Context, dir string) error
  • func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error)
  • func (f *Fs) Name() string
  • func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error)
  • func (f *Fs) Precision() time.Duration
  • func (f *Fs) Purge(ctx context.Context, dir string) error
  • func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
  • func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
  • func (f *Fs) Rmdir(ctx context.Context, dir string) error
  • func (f *Fs) Root() string
  • func (f *Fs) String() string
• type Object
  • func (o *Object) Fs() fs.Info
  • func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error)
  • func (o *Object) ModTime(ctx context.Context) time.Time
  • func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error)
  • func (o *Object) Remote() string
  • func (o *Object) Remove(ctx context.Context) error
  • func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error
  • func (o *Object) Size() int64
  • func (o *Object) Storable() bool
  • func (o *Object) String() string
  • func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error
• type Options

Functions

func NewFs

func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error)

NewFs constructs an Fs from the path

Types

type Fs

type Fs struct {
	// contains filtered or unexported fields
}

Fs represents a HDFS server

func (*Fs) About

func (f *Fs) About(ctx context.Context) (*fs.Usage, error)

About gets quota information from the Fs

func (*Fs) DirMove

func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error)

DirMove moves src, srcRemote to this remote at dstRemote using server-side move operations.

Will only be called if src.Fs().Name() == f.Name()

If it isn't possible then return fs.ErrorCantDirMove

If destination exists then return fs.ErrorDirExists

func (*Fs) Features

func (f *Fs) Features() *fs.Features

Features returns the optional features of this Fs

func (*Fs) Hashes

func (f *Fs) Hashes() hash.Set

Hashes are not supported

func (*Fs) List

func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error)

List the objects and directories in dir into entries.

func (*Fs) Mkdir

func (f *Fs) Mkdir(ctx context.Context, dir string) error

Mkdir makes a directory

func (*Fs) Move

func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error)

Move src to this remote using server-side move operations.

This is stored with the remote path given.

It returns the destination Object and a possible error.

Will only be called if src.Fs().Name() == f.Name()

If it isn't possible then return fs.ErrorCantMove

func (*Fs) Name

func (f *Fs) Name() string

Name of this fs

func (*Fs) NewObject

func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error)

NewObject finds file at remote or return fs.ErrorObjectNotFound

func (*Fs) Precision

func (f *Fs) Precision() time.Duration

Precision return the precision of this Fs

func (*Fs) Purge

func (f *Fs) Purge(ctx context.Context, dir string) error

Purge deletes all the files in the directory

func (*Fs) Put

func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

Put the object

func (*Fs) PutStream

func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

PutStream uploads to the remote path with the modTime given of indeterminate size

func (*Fs) Rmdir

func (f *Fs) Rmdir(ctx context.Context, dir string) error

Rmdir deletes the directory

func (*Fs) Root

func (f *Fs) Root() string

Root of the remote (as passed into NewFs)

func (*Fs) String

func (f *Fs) String() string

String returns a description of the FS

type Object

type Object struct {
	// contains filtered or unexported fields
}

Object describes an HDFS file

func (*Object) Fs

func (o *Object) Fs() fs.Info

Fs returns the parent Fs

func (*Object) Hash

func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error)

Hash is not supported

func (*Object) ModTime

func (o *Object) ModTime(ctx context.Context) time.Time

ModTime returns the modification time of the object

func (*Object) Open

func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error)

Open an object for read

func (*Object) Remote

func (o *Object) Remote() string

Remote returns the remote path

func (*Object) Remove

func (o *Object) Remove(ctx context.Context) error

Remove an object

func (*Object) SetModTime

func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error

SetModTime sets the modification time of the local fs object

func (*Object) Size

func (o *Object) Size() int64

Size returns the size of an object in bytes

func (*Object) Storable

func (o *Object) Storable() bool

Storable returns whether this object is storable

func (*Object) String

func (o *Object) String() string

Return a string version

func (*Object) Update

func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error

Update object

type Options

type Options struct {
	Namenode               string               `config:"namenode"`
	Username               string               `config:"username"`
	ServicePrincipalName   string               `config:"service_principal_name"`
	DataTransferProtection string               `config:"data_transfer_protection"`
	Enc                    encoder.MultiEncoder `config:"encoding"`
}

Options for this backend