files

Documentation

Overview

Package files implements an abstraction of accessing data via URL/URIs.

The basic usage of the library is fairly straight-forward:

import (
	"context"
	"fmt"
	"io"
	"os"

	"github.com/puellanivis/breton/lib/files"
	_ "github.com/puellanivis/breton/lib/files/plugins"
)

ctx = context.Background()

r, err := files.Open(ctx, "http://www.example.com")
if err != nil {
	return err
}
// copy the response.Body from the HTTP request to stdout.
if _, err := files.Copy(ctx, os.Stdout, r); err != nil {
	return err
}
r.Close()

w, err := files.Create(ctx, "clipboard:")
if err != nil {
	// will return `os.IsNotExist(err) == true` if not available.
	return err
}
fmt.Fprint(w, "write this string to the OS clipboard if available")
w.Close()

fi, err := files.List(ctx, "home:")
if err != nil {
	return err
}
for _, info := range fi {
	// will print each filename and size listed in the user.Current().HomeDir
	fmt.Printf("%s %d\n", info.Name(), info.Size())
}

Additionally, there are some helper functions which simply read/write a whole byte slice.

b, err := files.Read(ctx, "source")
if err != nil {
	return err
}
// no need to close here.

err := files.Write(ctx, "destination", b)
if err != nil {
	// will return io.ErrShortWrite if it does not write all of the buffer.
	return err
}
// no need to close here.

Or use io.ReaderCloser/io.WriteCloser as a source/destination

// discard and close all data on the io.ReadCloser
if err := files.Discard(io.ReadCloser); err != nil {
	return err
}
// no need to close here.

// read all the data on the io.ReadCloser into a byte-slice, then close the source.
b, err := files.ReadFrom(io.ReadCloser)
if err != nil {
	return err
}
// no need to close here.

// write all the data from the byte-slice into the io.WriteCloser, then close the source.
if err := files.WriteTo(io.WriteCloser, b); err != nil {
	return err
}
// no need to close here.

Index

• Variables
• func Copy(ctx context.Context, dst io.Writer, src io.Reader, opts ...CopyOption) (written int64, err error)
• func Discard(r io.Reader) error
• func GetRoot(ctx context.Context) (string, bool)
• func List(ctx context.Context, url string) ([]os.FileInfo, error)
• func NewInvalidURLError(reason string) error
• func PathError(op, path string, err error) error
• func Read(ctx context.Context, url string) ([]byte, error)
• func ReadDir(ctx context.Context, url string) ([]os.FileInfo, error)
• func ReadFrom(r io.Reader) ([]byte, error)
• func RegisterScheme(fs FileStore, schemes ...string)
• func RegisteredSchemes() []string
• func WithRoot(ctx context.Context, path string) (context.Context, error)
• func WithRootURL(ctx context.Context, uri *url.URL) context.Context
• func Write(ctx context.Context, url string, data []byte) error
• func WriteTo(w io.Writer, data []byte) error
• type CopyOption
  • func WithBandwidthMetrics(total interface{ ... }) CopyOption
  • func WithBuffer(buf []byte) CopyOption
  • func WithBufferSize(size int) CopyOption
  • func WithIntervalBandwidthMetrics(running interface{ ... }, n int, interval time.Duration) CopyOption
  • func WithMetricsScale(scale float64) CopyOption
  • func WithWatchdogTimeout(timeout time.Duration) CopyOption
• type File
• type FileStore
• type Option
  • func WithFileMode(mode os.FileMode) Option
• type Reader
  • func Open(ctx context.Context, url string, options ...Option) (Reader, error)
• type Writer
  • func Create(ctx context.Context, url string, options ...Option) (Writer, error)

Variables

var (
	// ErrNotSupported should be returned, if a particular feature or option is not supported.
	ErrNotSupported = errors.New("not supported")

	// ErrNotDirectory should be returned, if a request is made to ReadDir with a non-directory.
	ErrNotDirectory = syscall.ENOTDIR
)

var (
	ErrURLInvalid = NewInvalidURLError("invalid url")

	ErrURLCannotHaveAuthority = NewInvalidURLError("invalid url: scheme cannot have an authority")
	ErrURLCannotHaveHost      = NewInvalidURLError("invalid url: scheme cannot have a host in authority")

	ErrURLHostRequired = NewInvalidURLError("invalid url: scheme requires non-empty host in authority")
	ErrURLPathRequired = NewInvalidURLError("invalid url: scheme requires non-empty path")
)

A set of Invalid URL Error to better identify and relate specific invalid URL details.

var ErrWatchdogExpired error = watchdogExpiredError{}

ErrWatchdogExpired is returned by files.Copy, if the watchdog time expires during a read.

Functions

func Copy

func Copy(ctx context.Context, dst io.Writer, src io.Reader, opts ...CopyOption) (written int64, err error)

Copy is a context aware version of io.Copy. Do not use to Discard a reader, as a canceled context would stop the read, and it would not be fully discarded.

func Discard

func Discard(r io.Reader) error

Discard throws away the entire content of an io.Reader. If the Reader also implements io.Closer, it will also Close it.

This is specifically not context aware, it is intended to always run to completion.

func GetRoot

func GetRoot(ctx context.Context) (string, bool)

GetRoot returns the currently attached string that is being used as the root for any invalid URLs.

func List

func List(ctx context.Context, url string) ([]os.FileInfo, error)

List reads the directory or listing of the resource at the given URL, and returns a slice of os.FileInfo, which describe the files contained in the directory.

Depcrecated: Use `ReadDir`.

func NewInvalidURLError

func NewInvalidURLError(reason string) error

NewInvalidURLError returns an error that formats as the given text, and where errors.Is will return true for both: files.ErrURLInvalid and os.ErrInvalid.

func PathError

func PathError(op, path string, err error) error

PathError is DEPRECATED, and returns an *os.PathError with appropriate fields set. DO NOT USE.

This is a stop-gap quick-replace to remove `&os.PathError{ op, path, err }`. One should use the direct complex literal instruction instead.

Deprecated: use &os.PathError{} directly.

func Read

func Read(ctx context.Context, url string) ([]byte, error)

Read reads the entire content of the resource at the given URL into a byte-slice.

func ReadDir

func ReadDir(ctx context.Context, url string) ([]os.FileInfo, error)

ReadDir reads the directory or listing of the resource at the given URL, and returns a slice of os.FileInfo, which describe the files contained in the directory.

func ReadFrom

func ReadFrom(r io.Reader) ([]byte, error)

ReadFrom reads the entire content of an io.Reader and returns the content as a byte slice. If the Reader also implements io.Closer, it will also Close it.

func RegisterScheme

func RegisterScheme(fs FileStore, schemes ...string)

RegisterScheme takes a FileStore and attaches to it the given schemes so that files.Open will use that FileStore when a files.Open() is performed with a URL of any of those schemes.

func RegisteredSchemes

func RegisteredSchemes() []string

RegisteredSchemes returns a slice of strings that describe all registered schemes.

func WithRoot

func WithRoot(ctx context.Context, path string) (context.Context, error)

WithRoot stores either a URL or a local path to use as a root point when resolving filenames.

func WithRootURL

func WithRootURL(ctx context.Context, uri *url.URL) context.Context

WithRootURL attaches a url.URL to a Context and is used as the resolution reference for any files.Open() using that context.

func Write

func Write(ctx context.Context, url string, data []byte) error

Write writes the entire content of data to the resource at the given URL.

func WriteTo

func WriteTo(w io.Writer, data []byte) error

WriteTo writes the entire content of data to an io.Writer. If the Writer also implements io.Closer, it will also Close it.

Types

type CopyOption

type CopyOption func(c *copyConfig) CopyOption

CopyOption defines a function that applies a value or setting for a specific files.Copy operation.

func WithBandwidthMetrics

func WithBandwidthMetrics(total interface{ Observe(float64) }) CopyOption

WithBandwidthMetrics establishes a lifetime bandwidth metric for the files.Copy.

func WithBuffer

func WithBuffer(buf []byte) CopyOption

WithBuffer sets which buffer a files.Copy should use internally as temporary storage between the Read and Write.

func WithBufferSize

func WithBufferSize(size int) CopyOption

WithBufferSize makes a new buffer of size bytes, which is used as temporary intermediate storage for the files.Copy.

func WithIntervalBandwidthMetrics

func WithIntervalBandwidthMetrics(running interface{ Observe(float64) }, n int, interval time.Duration) CopyOption

WithIntervalBandwidthMetrics keeps a running list of n intervals and reports the bandwidth over this window.

func WithMetricsScale

func WithMetricsScale(scale float64) CopyOption

WithMetricsScale sets the scale of reported Metrics, otherwise it is reported in bytes/second.

func WithWatchdogTimeout

func WithWatchdogTimeout(timeout time.Duration) CopyOption

WithWatchdogTimeout sets a running interval timeout, where if no copy progress is made during that time, the files.Copy will fail with a timeout error.

type File

type File interface {
	io.Closer
	Name() string
	Stat() (os.FileInfo, error)
}

File defines an interface that abstracts the concept of files to allow for multiple types implementation, beyond the local filesystem.

type FileStore

type FileStore interface {
	Open(ctx context.Context, uri *url.URL) (Reader, error)
	Create(ctx context.Context, uri *url.URL) (Writer, error)
	List(ctx context.Context, uri *url.URL) ([]os.FileInfo, error)
}

FileStore defines an interface which implements a system of accessing files for reading (Open) writing (Write) and directly listing (List)

var Local FileStore = &localFS{}

Local implements a wrapper from the os functions Open, Create, and Readdir, to the files.FileStore implementation.

type Option

type Option func(File) (Option, error)

Option is a function that applies a specific option to a files.File, it returns an Option and and error. If error is not nil, then the Option returned will revert the option that was set. Since errors returned by Option arguments are discarded by Open(), and Create(), if you care about the error status of an Option you must apply it yourself after Open() or Create()

func WithFileMode

func WithFileMode(mode os.FileMode) Option

WithFileMode returns an Option that will set the files.File.Stat().FileMode() to the given os.FileMode.

type Reader

type Reader interface {
	File
	io.ReadSeeker
}

Reader defines a files.File that is also an io.ReadSeeker

func Open

func Open(ctx context.Context, url string, options ...Option) (Reader, error)

Open returns a files.Reader, which can be used to read content from the resource at the given URL.

All errors and reversion functions returned by Option arguments are discarded.

type Writer

type Writer interface {
	File
	io.Writer
	Sync() error
}

Writer defines a files.File that is also an io.Writer with a Sync() function.

func Create

func Create(ctx context.Context, url string, options ...Option) (Writer, error)

Create returns a files.Writer, which can be used to write content to the resource at the given URL.

If the given URL is a local filename, the file will be created, and truncated before this function returns.

All errors and reversion functions returned by Option arguments are discarded.