siva

README

śiva format शिव

śiva stand for seekable indexed block archiver

śiva is archive format very similar to tar or zip, focused on allowing: constant-time random file access, seekable access to the contained files and concatenable archive files

The library implements a very similar API to the go tar package, allowing full control over and low level access to the contained files.

• Library reference
• Command-line interface
• Format specification

Installation

The recommended way to install siva

go get -u gopkg.in/src-d/go-siva.v1/...

Example

Creating a siva file:

// Create a buffer to write our archive to.
buf := new(bytes.Buffer)

// Create a new siva archive.
w := siva.NewWriter(buf)

// Add some files to the archive.
var files = []struct {
    Name, Body string
}{
    {"readme.txt", "This archive contains some text files."},
    {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
    {"todo.txt", "Get animal handling license."},
}
for _, file := range files {
    hdr := &siva.Header{
        Name:    file.Name,
        Mode:    0600,
        ModTime: time.Now(),
    }
    if err := w.WriteHeader(hdr); err != nil {
        log.Fatalln(err)
    }
    if _, err := w.Write([]byte(file.Body)); err != nil {
        log.Fatalln(err)
    }
}
// Make sure to check the error on Close.
if err := w.Close(); err != nil {
    log.Fatalln(err)
}

Reading from a siva file:

// Open the siva archive for reading.
file := bytes.NewReader(buf.Bytes())
r := siva.NewReader(file)

// Get all the files in the siva file.
i, err := r.Index()
if err != nil {
    log.Fatalln(err)
}

// Iterate through the files in the archive.
for _, e := range i {
    content, err := r.Get(e)
    if err != nil {
        log.Fatalln(err)
    }
    fmt.Printf("Contents of %s:\n", e.Name)
    if _, err := io.Copy(os.Stdout, content); err != nil {
        log.Fatalln(err)
    }
    fmt.Println()
}

Command-line interface

siva cli interface, is a convenient command that helps you to creates and manipulates siva files.

Output from: ./siva --help:

Usage:
  siva [OPTIONS] <command>

Help Options:
  -h, --help  Show this help message

Available commands:
  list     List the items contained on a file.
  pack     Create a new archive containing the specified items.
  unpack   Extract to disk from the archive.
  version  Show the version information.

License

MIT, see LICENSE

Documentation

Overview

Example

// Create a buffer to write our archive to.
buf := new(bytes.Buffer)

// Create a new siva archive.
w := siva.NewWriter(buf)

// Add some files to the archive.
var files = []struct {
	Name, Body string
}{
	{"readme.txt", "This archive contains some text files."},
	{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
	{"todo.txt", "Get animal handling license."},
}
for _, file := range files {
	hdr := &siva.Header{
		Name:    file.Name,
		Mode:    0600,
		ModTime: time.Now(),
	}
	if err := w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err := w.Write([]byte(file.Body)); err != nil {
		log.Fatalln(err)
	}
}
// Make sure to check the error on Close.
if err := w.Close(); err != nil {
	log.Fatalln(err)
}

// Open the siva archive for reading.
file := bytes.NewReader(buf.Bytes())
r := siva.NewReader(file)

// Get all files in the siva file.
i, err := r.Index()
if err != nil {
	log.Fatalln(err)
}

// Iterate through the files in the archive.
for _, e := range i {
	content, err := r.Get(e)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Printf("Contents of %s:\n", e.Name)
	if _, err := io.Copy(os.Stdout, content); err != nil {
		log.Fatalln(err)
	}
	fmt.Println()
}

Output:
Contents of readme.txt:
This archive contains some text files.
Contents of gopher.txt:
Gopher names:
George
Geoffrey
Gonzo
Contents of todo.txt:
Get animal handling license.

Index

• Constants
• Variables
• type Flag
• type Header
• type Index
  • func (i *Index) Filter() Index
  • func (i Index) Find(name string) *IndexEntry
  • func (i Index) Glob(pattern string) ([]*IndexEntry, error)
  • func (s Index) Len() int
  • func (s Index) Less(i, j int) bool
  • func (i *Index) ReadFrom(r io.ReadSeeker, endBlock uint64) error
  • func (s Index) Swap(i, j int)
  • func (i *Index) WriteTo(w io.Writer) error
• type IndexEntry
  • func (e *IndexEntry) ReadFrom(r io.Reader) error
  • func (e *IndexEntry) WriteTo(w io.Writer) error
• type IndexFooter
  • func (f *IndexFooter) ReadFrom(r io.Reader) error
  • func (f *IndexFooter) WriteTo(w io.Writer) error
• type IndexReadError
  • func (e *IndexReadError) Error() string
• type IndexWriteError
  • func (e *IndexWriteError) Error() string
• type ReadWriter
  • func NewReaderWriter(rw io.ReadWriteSeeker) (*ReadWriter, error)
  • func (w ReadWriter) Close() error
  • func (w ReadWriter) Flush() error
  • func (r ReadWriter) Get(e *IndexEntry) (*io.SectionReader, error)
  • func (r ReadWriter) Index() (Index, error)
  • func (r ReadWriter) Read(p []byte) (n int, err error)
  • func (r ReadWriter) Seek(e *IndexEntry) (int64, error)
  • func (w ReadWriter) Write(b []byte) (int, error)
  • func (w ReadWriter) WriteHeader(h *Header) error
• type Reader
  • func NewReader(r io.ReadSeeker) Reader
• type Writer
  • func NewWriter(w io.Writer) Writer

Examples

• Package

Constants

const (
	IndexVersion uint8 = 1
)

Variables

var (
	IndexSignature = []byte{'I', 'B', 'A'}

	ErrInvalidIndexEntry       = errors.New("invalid index entry")
	ErrInvalidSignature        = errors.New("invalid signature")
	ErrEmptyIndex              = errors.New("empty index")
	ErrUnsupportedIndexVersion = errors.New("unsupported index version")
	ErrCRC32Missmatch          = errors.New("crc32 missmatch")
)

var (
	ErrPendingContent   = errors.New("entry wasn't fully read")
	ErrInvalidCheckshum = errors.New("invalid checksum")
	ErrInvalidReaderAt  = errors.New("reader provided dosen't implement ReaderAt interface")
)

var (
	ErrMissingHeader = errors.New("WriteHeader was not called, or already flushed")
	ErrClosedWriter  = errors.New("Writer is closed")
)

Types

type Flag

type Flag uint32

const (

	//FlagDeleted should be used to identify when a file is deleted
	FlagDeleted Flag = iota
)

type Header

type Header struct {
	Name    string
	ModTime time.Time
	Mode    os.FileMode
	Flags   Flag
}

Header contains the meta information from a file

type Index

type Index []*IndexEntry

Index contains all the files on a siva file, including duplicate files or even does flagged as deleted.

func (*Index) Filter

func (i *Index) Filter() Index

Filter returns a filtered version of the current Index removing duplicates keeping the latest versions and filtering all the deleted files

func (Index) Find

func (i Index) Find(name string) *IndexEntry

Find returns the first IndexEntry with the given name, if any

func (Index) Glob

func (i Index) Glob(pattern string) ([]*IndexEntry, error)

Glob returns all index entries whose name matches pattern or nil if there is no matching entry. The syntax of patterns is the same as in filepath.Match.

func (Index) Len

func (s Index) Len() int

func (Index) Less

func (s Index) Less(i, j int) bool

func (*Index) ReadFrom

func (i *Index) ReadFrom(r io.ReadSeeker, endBlock uint64) error

ReadFrom reads an Index from a given reader, the position where the current block ends is required since we are reading the index from the end of the file

func (Index) Swap

func (s Index) Swap(i, j int)

func (*Index) WriteTo

func (i *Index) WriteTo(w io.Writer) error

WriteTo writes the Index to a io.Writer

type IndexEntry

type IndexEntry struct {
	Header
	Start uint64
	Size  uint64
	CRC32 uint32
	// contains filtered or unexported fields
}

func (*IndexEntry) ReadFrom

func (e *IndexEntry) ReadFrom(r io.Reader) error

ReadFrom reads a IndexEntry entry from an io.Reader

func (*IndexEntry) WriteTo

func (e *IndexEntry) WriteTo(w io.Writer) error

WriteTo writes the IndexEntry to an io.Writer

type IndexFooter

type IndexFooter struct {
	EntryCount uint32
	IndexSize  uint64
	BlockSize  uint64
	CRC32      uint32
}

func (*IndexFooter) ReadFrom

func (f *IndexFooter) ReadFrom(r io.Reader) error

ReadFrom reads a IndexFooter entry from an io.Reader

func (*IndexFooter) WriteTo

func (f *IndexFooter) WriteTo(w io.Writer) error

WriteTo writes the IndexFooter to an io.Writer

type IndexReadError

type IndexReadError struct {
	Err error
}

func (*IndexReadError) Error

func (e *IndexReadError) Error() string

type IndexWriteError

type IndexWriteError struct {
	Err error
}

func (*IndexWriteError) Error

func (e *IndexWriteError) Error() string

type ReadWriter

type ReadWriter struct {
	// contains filtered or unexported fields
}

ReadWriter can read and write to the same siva file. It is not thread-safe.

func NewReaderWriter

func NewReaderWriter(rw io.ReadWriteSeeker) (*ReadWriter, error)

func (ReadWriter) Close

func (w ReadWriter) Close() error

Close closes the siva archive, writing the Index footer to the current writer.

func (ReadWriter) Flush

func (w ReadWriter) Flush() error

Flush finishes writing the current file (optional)

func (ReadWriter) Get

func (r ReadWriter) Get(e *IndexEntry) (*io.SectionReader, error)

Get returns a new io.SectionReader allowing concurrent read access to the content of the read

func (ReadWriter) Index

func (r ReadWriter) Index() (Index, error)

Index reads the index of the siva file from the provided reader

func (ReadWriter) Read

func (r ReadWriter) Read(p []byte) (n int, err error)

Read reads up to len(p) bytes, starting at the current position set by Seek and ending in the end of the content, retuning a io.EOF when its reached

func (ReadWriter) Seek

func (r ReadWriter) Seek(e *IndexEntry) (int64, error)

Seek seek the internal reader to the starting position of the content for the given IndexEntry

func (ReadWriter) Write

func (w ReadWriter) Write(b []byte) (int, error)

Write writes to the current entry in the siva archive, WriteHeader should called before, if not returns ErrMissingHeader

func (ReadWriter) WriteHeader

func (w ReadWriter) WriteHeader(h *Header) error

WriteHeader writes hdr and prepares to accept the file's contents.

type Reader

type Reader interface {
	io.Reader
	Seek(e *IndexEntry) (int64, error)
	Index() (Index, error)
	Get(e *IndexEntry) (*io.SectionReader, error)
}

A Reader provides random access to the contents of a siva archive.

func NewReader

func NewReader(r io.ReadSeeker) Reader

NewReader creates a new Reader reading from r, reader requires be seekable and optionally should implement io.ReaderAt to make usage of the Get method

type Writer

type Writer interface {
	io.Writer
	io.Closer
	WriteHeader(h *Header) error
	Flush() error
}

A Writer provides sequential writing of a siva archive

func NewWriter

func NewWriter(w io.Writer) Writer

NewWriter creates a new Writer writing to w.