internetarchive

Documentation

Overview

Package internetarchive provides an interface to Internet Archive's Item via their native API than using S3-compatible endpoints.

Index

• func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error)
• type Fs
  • func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error)
  • func (f *Fs) CleanUp(ctx context.Context) (err error)
  • func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Object, err error)
  • func (f *Fs) Features() *fs.Features
  • func (f *Fs) Hashes() hash.Set
  • func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error)
  • func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error)
  • func (f *Fs) Mkdir(ctx context.Context, dir string) (err error)
  • func (f *Fs) Name() string
  • func (f *Fs) NewObject(ctx context.Context, remote string) (ret fs.Object, err error)
  • func (f *Fs) Precision() time.Duration
  • func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error)
  • func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
  • func (f *Fs) Rmdir(ctx context.Context, dir string) error
  • func (f *Fs) Root() string
  • func (f *Fs) String() string
• type IAFile
• type MetadataResponse
• type MetadataResponseRaw
• type ModMetadataResponse
• type Object
  • func (o *Object) Fs() fs.Info
  • func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error)
  • func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error)
  • func (o *Object) ModTime(ctx context.Context) time.Time
  • func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error)
  • func (o *Object) Remote() string
  • func (o *Object) Remove(ctx context.Context) (err error)
  • func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error)
  • func (o *Object) Size() int64
  • func (o *Object) Storable() bool
  • func (o *Object) String() string
  • func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error)
• type Options

Functions

func NewFs

func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error)

NewFs constructs an Fs from the path

Types

type Fs

type Fs struct {
	// contains filtered or unexported fields
}

Fs represents an IAS3 remote

func (*Fs) About

func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error)

About returns things about remaining and used spaces

func (*Fs) CleanUp

func (f *Fs) CleanUp(ctx context.Context) (err error)

CleanUp removes all files inside history/

func (*Fs) Copy

func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Object, err error)

Copy src to this remote using server-side copy operations.

This is stored with the remote path given.

It returns the destination Object and a possible error.

Will only be called if src.Fs().Name() == f.Name()

If it isn't possible then return fs.ErrorCantCopy

func (*Fs) Features

func (f *Fs) Features() *fs.Features

Features returns the optional features of this Fs

func (*Fs) Hashes

func (f *Fs) Hashes() hash.Set

Hashes returns type of hashes supported by IA

func (*Fs) List

func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error)

List files and directories in a directory

func (*Fs) ListR

func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error)

ListR lists the objects and directories of the Fs starting from dir recursively into out.

dir should be "" to start from the root, and should not have trailing slashes.

This should return ErrDirNotFound if the directory isn't found.

It should call callback for each tranche of entries read. These need not be returned in any particular order. If callback returns an error then the listing will stop immediately.

Don't implement this unless you have a more efficient way of listing recursively than doing a directory traversal.

func (*Fs) Mkdir

func (f *Fs) Mkdir(ctx context.Context, dir string) (err error)

Mkdir can't be performed on IA like git repositories

func (*Fs) Name

func (f *Fs) Name() string

Name of the remote (as passed into NewFs)

func (*Fs) NewObject

func (f *Fs) NewObject(ctx context.Context, remote string) (ret fs.Object, err error)

NewObject finds the Object at remote. If it can't be found it returns the error fs.ErrorObjectNotFound.

func (*Fs) Precision

func (f *Fs) Precision() time.Duration

Precision returns the precision of mtime that the server responds

func (*Fs) PublicLink

func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error)

PublicLink generates a public link to the remote path (usually readable by anyone)

func (*Fs) Put

func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)

Put uploads a file

func (*Fs) Rmdir

func (f *Fs) Rmdir(ctx context.Context, dir string) error

Rmdir as well, unless we're asked for recursive deletion

func (*Fs) Root

func (f *Fs) Root() string

Root of the remote (as passed into NewFs)

func (*Fs) String

func (f *Fs) String() string

String converts this Fs to a string

type IAFile

type IAFile struct {
	Name string `json:"name"`
	// Source     string `json:"source"`
	Mtime       string          `json:"mtime"`
	RcloneMtime json.RawMessage `json:"rclone-mtime"`
	UpdateTrack json.RawMessage `json:"rclone-update-track"`
	Size        string          `json:"size"`
	Md5         string          `json:"md5"`
	Crc32       string          `json:"crc32"`
	Sha1        string          `json:"sha1"`
	Summation   string          `json:"summation"`
	// contains filtered or unexported fields
}

IAFile represents a subset of object in MetadataResponse.Files

type MetadataResponse

type MetadataResponse struct {
	Files    []IAFile `json:"files"`
	ItemSize int64    `json:"item_size"`
}

MetadataResponse represents subset of the JSON object returned by (frontend)/metadata/

type MetadataResponseRaw

type MetadataResponseRaw struct {
	Files    []json.RawMessage `json:"files"`
	ItemSize int64             `json:"item_size"`
}

MetadataResponseRaw is the form of MetadataResponse to deal with metadata

type ModMetadataResponse

type ModMetadataResponse struct {
	// https://archive.org/services/docs/api/md-write.html#example
	Success bool   `json:"success"`
	Error   string `json:"error"`
}

ModMetadataResponse represents response for amending metadata

type Object

type Object struct {
	// contains filtered or unexported fields
}

Object describes a file at IA

func (*Object) Fs

func (o *Object) Fs() fs.Info

Fs returns the parent Fs

func (*Object) Hash

func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error)

Hash returns the hash value presented by IA

func (*Object) Metadata

func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error)

Metadata returns all file metadata provided by Internet Archive

func (*Object) ModTime

func (o *Object) ModTime(ctx context.Context) time.Time

ModTime is the last modified time (read-only)

func (*Object) Open

func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error)

Open an object for read

func (*Object) Remote

func (o *Object) Remote() string

Remote returns the remote path

func (*Object) Remove

func (o *Object) Remove(ctx context.Context) (err error)

Remove an object

func (*Object) SetModTime

func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error)

SetModTime sets modTime on a particular file

func (*Object) Size

func (o *Object) Size() int64

Size is the file length

func (*Object) Storable

func (o *Object) Storable() bool

Storable returns if this object is storable

func (*Object) String

func (o *Object) String() string

String converts this Fs to a string

func (*Object) Update

func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error)

Update the Object from in with modTime and size

type Options

type Options struct {
	AccessKeyID     string               `config:"access_key_id"`
	SecretAccessKey string               `config:"secret_access_key"`
	Endpoint        string               `config:"endpoint"`
	FrontEndpoint   string               `config:"front_endpoint"`
	DisableChecksum bool                 `config:"disable_checksum"`
	WaitArchive     fs.Duration          `config:"wait_archive"`
	Enc             encoder.MultiEncoder `config:"encoding"`
}

Options defines the configuration for this backend