siva

README

śiva format शिव [] (https://travis-ci.org/src-d/go-siva)

śiva stand for seekable indexed block archiver

śiva is archive format very similar to tar or zip, focused on allowing: constant-time random file access, seekable access to the contained files and concatenable archive files

The library implements a very similar API to the go tar package, allowing full control over and low level access to the contained files.

• Library reference
• Command-line interface

Installation

The recommended way to install siva

go get -u github.com/src-d/go-siva/...

Example

Creating a siva file:

// Create a buffer to write our archive to.
buf := new(bytes.Buffer)

// Create a new siva archive.
w := siva.NewWriter(buf)

// Add some files to the archive.
var files = []struct {
    Name, Body string
}{
    {"readme.txt", "This archive contains some text files."},
    {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
    {"todo.txt", "Get animal handling license."},
}
for _, file := range files {
    hdr := &siva.Header{
        Name:    file.Name,
        Mode:    0600,
        ModTime: time.Now(),
    }
    if err := w.WriteHeader(hdr); err != nil {
        log.Fatalln(err)
    }
    if _, err := w.Write([]byte(file.Body)); err != nil {
        log.Fatalln(err)
    }
}
// Make sure to check the error on Close.
if err := w.Close(); err != nil {
    log.Fatalln(err)
}

Reading from a siva file:

// Open the siva archive for reading.
file := bytes.NewReader(buf.Bytes())
r := siva.NewReader(file)

// Get all the files in the siva file.
i, err := r.Index()
if err != nil {
    log.Fatalln(err)
}

// Iterate through the files in the archive.
for _, e := range i {
    content, err := r.Get(e)
    if err != nil {
        log.Fatalln(err)
    }
    fmt.Printf("Contents of %s:\n", e.Name)
    if _, err := io.Copy(os.Stdout, content); err != nil {
        log.Fatalln(err)
    }
    fmt.Println()
}

Command-line interface

siva cli interface, is a convenient command that helps you to creates and manipulates siva files.

Output from: ./siva --help:

Usage:
  siva [OPTIONS] <command>

Help Options:
  -h, --help  Show this help message

Available commands:
  list     List the items contained on a file.
  pack     Create a new archive containing the specified items.
  unpack   Extract to disk from the archive.
  version  Show the version information.

License

MIT, see LICENSE

Documentation

Overview

Example

// Create a buffer to write our archive to.
buf := new(bytes.Buffer)

// Create a new siva archive.
w := siva.NewWriter(buf)

// Add some files to the archive.
var files = []struct {
	Name, Body string
}{
	{"readme.txt", "This archive contains some text files."},
	{"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
	{"todo.txt", "Get animal handling license."},
}
for _, file := range files {
	hdr := &siva.Header{
		Name:    file.Name,
		Mode:    0600,
		ModTime: time.Now(),
	}
	if err := w.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
	if _, err := w.Write([]byte(file.Body)); err != nil {
		log.Fatalln(err)
	}
}
// Make sure to check the error on Close.
if err := w.Close(); err != nil {
	log.Fatalln(err)
}

// Open the siva archive for reading.
file := bytes.NewReader(buf.Bytes())
r := siva.NewReader(file)

// Get all the files in the siva file.
i, err := r.Index()
if err != nil {
	log.Fatalln(err)
}

// Iterate through the files in the archive.
for _, e := range i {
	content, err := r.Get(e)
	if err != nil {
		log.Fatalln(err)
	}
	fmt.Printf("Contents of %s:\n", e.Name)
	if _, err := io.Copy(os.Stdout, content); err != nil {
		log.Fatalln(err)
	}
	fmt.Println()
}

Output:

Contents of readme.txt:
This archive contains some text files.
Contents of gopher.txt:
Gopher names:
George
Geoffrey
Gonzo
Contents of todo.txt:
Get animal handling license.

Index

• Constants
• Variables
• type Flag
• type Header
• type Index
  • func (i *Index) Filter() Index
  • func (i Index) Find(name string) *IndexEntry
  • func (s Index) Len() int
  • func (s Index) Less(i, j int) bool
  • func (i *Index) ReadFrom(r io.ReadSeeker, endBlock uint64) error
  • func (s Index) Swap(i, j int)
  • func (i *Index) WriteTo(w io.Writer) error
• type IndexEntry
  • func (e *IndexEntry) ReadFrom(r io.Reader) error
  • func (e *IndexEntry) WriteTo(w io.Writer) error
• type IndexFooter
  • func (f *IndexFooter) ReadFrom(r io.Reader) error
  • func (f *IndexFooter) WriteTo(w io.Writer) error
• type Reader
  • func NewReader(r io.ReadSeeker) *Reader
  • func (r *Reader) Get(e *IndexEntry) (*io.SectionReader, error)
  • func (r *Reader) Index() (Index, error)
  • func (r *Reader) Read(p []byte) (n int, err error)
  • func (r *Reader) Seek(e *IndexEntry) (int64, error)
• type Writer
  • func NewWriter(w io.Writer) *Writer
  • func (w *Writer) Close() error
  • func (w *Writer) Flush() error
  • func (w *Writer) Write(b []byte) (int, error)
  • func (w *Writer) WriteHeader(h *Header) error

Examples

• Package

Constants

const (
	IndexVersion uint8 = 1
)

Variables

var (
	IndexSignature = []byte{'I', 'B', 'A'}

	ErrInvalidIndexEntry       = errors.New("invalid index entry")
	ErrInvalidSignature        = errors.New("invalid signature")
	ErrEmptyIndex              = errors.New("empty index")
	ErrUnsupportedIndexVersion = errors.New("unsupported index version")
	ErrCRC32Missmatch          = errors.New("crc32 missmatch")
)

var (
	ErrPendingContent   = errors.New("entry wasn't fully read")
	ErrInvalidCheckshum = errors.New("invalid checksum")
	ErrInvalidReaderAt  = errors.New("reader provided doen't implements ReaderAt interface")
)

var (
	ErrMissingHeader = errors.New("WriteHeader was not called, or already flushed")
	ErrClosedWriter  = errors.New("Writer is closed")
)

Types

type Flag

type Flag uint32

const (

	//FlagDeleted should be used to identify when a file is deleted
	FlagDeleted Flag = iota
)

type Header

type Header struct {
	Name    string
	ModTime time.Time
	Mode    os.FileMode
	Flags   Flag
}

Header contains the meta information from a file

type Index

type Index []*IndexEntry

Index contains all the files on a siva file, including duplicate files or even does flagged as deleted

func (*Index) Filter

func (i *Index) Filter() Index

Filter returns a filtered version of the current Index removing duplicates keeping the latests versions and filtering all the deleted files

func (Index) Find

func (i Index) Find(name string) *IndexEntry

Find returns the first IndexEntry with the given name, if any

func (Index) Len

func (s Index) Len() int

func (Index) Less

func (s Index) Less(i, j int) bool

func (*Index) ReadFrom

func (i *Index) ReadFrom(r io.ReadSeeker, endBlock uint64) error

ReadFrom reads an Index from a given reader, the position where the current block ends is required since we are reading the index from the end of the file

func (Index) Swap

func (s Index) Swap(i, j int)

func (*Index) WriteTo

func (i *Index) WriteTo(w io.Writer) error

WriteTo writes the Index to a io.Writer

type IndexEntry

type IndexEntry struct {
	Header
	Start uint64
	Size  uint64
	CRC32 uint32
	// contains filtered or unexported fields
}

func (*IndexEntry) ReadFrom

func (e *IndexEntry) ReadFrom(r io.Reader) error

ReadFrom reads a IndexEntry entry from an io.Reader

func (*IndexEntry) WriteTo

func (e *IndexEntry) WriteTo(w io.Writer) error

WriteTo writes the IndexEntry to an io.Writer

type IndexFooter

type IndexFooter struct {
	EntryCount uint32
	IndexSize  uint64
	BlockSize  uint64
	CRC32      uint32
}

func (*IndexFooter) ReadFrom

func (f *IndexFooter) ReadFrom(r io.Reader) error

ReadFrom reads a IndexFooter entry from an io.Reader

func (*IndexFooter) WriteTo

func (f *IndexFooter) WriteTo(w io.Writer) error

WriteTo writes the IndexFooter to an io.Writer

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

A Reader provides random access to the contents of a siva archive.

func NewReader

func NewReader(r io.ReadSeeker) *Reader

NewReader creates a new Reader reading from r, reader requires be seekable and optionally should implement io.ReaderAt to make usage of the Get method

func (*Reader) Get

func (r *Reader) Get(e *IndexEntry) (*io.SectionReader, error)

Get returns a new io.SectionReader allowing concurrent read access to the content of the read

func (*Reader) Index

func (r *Reader) Index() (Index, error)

Index reads the index of the siva file from the provided reader

func (*Reader) Read

func (r *Reader) Read(p []byte) (n int, err error)

Read reads up to len(p) bytes, starting at the current position set by Seek and ending in the end of the content, retuning a io.EOF when its reached

func (*Reader) Seek

func (r *Reader) Seek(e *IndexEntry) (int64, error)

Seek seek the internal reader to the starting position of the content for the given IndexEntry

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

A Writer provides sequential writing of a siva archive

func NewWriter

func NewWriter(w io.Writer) *Writer

NewWriter creates a new Writer writing to w.

func (*Writer) Close

func (w *Writer) Close() error

Close closes the siva archive, writing the Index footer to the current writer.

func (*Writer) Flush

func (w *Writer) Flush() error

Flush finishes writing the current file (optional)

func (*Writer) Write

func (w *Writer) Write(b []byte) (int, error)

Write writes to the current entry in the siva archive, WriteHeader should called before, if not returns ErrMissingHeader

func (*Writer) WriteHeader

func (w *Writer) WriteHeader(h *Header) error

WriteHeader writes hdr and prepares to accept the file's contents.