extsort

README

ExtSort

External merge sort algorithm, implemented in Go. Sort arbitrarily large data sets with a predictable amount of memory using disk.

Example:

import(
  "fmt"

  "github.com/bsm/extsort"
)

func main() {
	// Create sorter.
	sorter := extsort.New(nil)
	defer sorter.Close()

	// Append data.
	_ = sorter.Append([]byte("foo"))
	_ = sorter.Append([]byte("bar"))
	_ = sorter.Append([]byte("baz"))
	_ = sorter.Append([]byte("dau"))

	// Sort and iterate.
	iter, err := sorter.Sort()
	if err != nil {
		panic(err)
	}
	defer iter.Close()

	for iter.Next() {
		fmt.Println(string(iter.Data()))
	}
	if err := iter.Err(); err != nil {
		panic(err)
	}

}

Documentation

Overview

Example

package main

import (
	"fmt"

	"github.com/bsm/extsort"
)

func main() {
	// Create sorter.
	sorter := extsort.New(nil)
	defer sorter.Close()

	// Append data.
	_ = sorter.Append([]byte("foo"))
	_ = sorter.Append([]byte("bar"))
	_ = sorter.Append([]byte("baz"))
	_ = sorter.Append([]byte("dau"))

	// Sort and iterate.
	iter, err := sorter.Sort()
	if err != nil {
		panic(err)
	}
	defer iter.Close()

	for iter.Next() {
		fmt.Println(string(iter.Data()))
	}
	if err := iter.Err(); err != nil {
		panic(err)
	}

}

Output:
bar
baz
dau
foo

Index

• type Compression
• type Iterator
  • func (i *Iterator) Close() error
  • func (i *Iterator) Data() []byte
  • func (i *Iterator) Err() error
  • func (i *Iterator) Next() bool
• type Less
• type Options
• type Sorter
  • func New(opt *Options) *Sorter
  • func (s *Sorter) Append(data []byte) error
  • func (s *Sorter) Close() error
  • func (s *Sorter) Sort() (*Iterator, error)

Examples

• Package

Types

type Compression

type Compression uint8

Compression codec.

const (
	CompressionNone Compression = iota
	CompressionGzip
)

Supported compression types.

type Iterator

type Iterator struct {
	// contains filtered or unexported fields
}

Iterator instances are used to iterate over sorted output.

func (*Iterator) Close

func (i *Iterator) Close() error

Close closes the iterator.

func (*Iterator) Data

func (i *Iterator) Data() []byte

Data returns the data at the current cursor position.

func (*Iterator) Err

func (i *Iterator) Err() error

Err returns the error, if occurred.

func (*Iterator) Next

func (i *Iterator) Next() bool

Next advances the iterator to the next item and returns true if successful.

type Less

type Less func(a, b []byte) bool

Less compares byte chunks.

type Options

type Options struct {
	// WorkDir specifies the working directory.
	// By default os.TempDir() is used.
	WorkDir string

	// Less defines the compare function.
	// Default: bytes.Compare() < 0
	Less Less

	// BufferSize limits the memory buffer used for sorting.
	// Default: 64MiB (must be at least 64KiB)
	BufferSize int

	// Compression optionally uses compression for temporary output.
	Compression Compression
}

Options contains sorting options

type Sorter

type Sorter struct {
	// contains filtered or unexported fields
}

Sorter is responsible for sorting.

func New

func New(opt *Options) *Sorter

New inits a sorter

func (*Sorter) Append

func (s *Sorter) Append(data []byte) error

Append appends a data chunk to the sorter.

func (*Sorter) Close

func (s *Sorter) Close() error

Close stops the processing and removes temporary files.

func (*Sorter) Sort

func (s *Sorter) Sort() (*Iterator, error)

Sort applies the sort algorithm and returns an interator.