Help us shape the future of Go! Take the 2026 Go Developer Survey

bakpack

package module
v0.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 24, 2026 License: MIT Imports: 23 Imported by: 0

README

bakpack

bakpack is a Go command line tool and library for compressing Bakta JSON annotation files while keeping them reconstructable and checkable against the original JSON content.

This repository was developed with substantial coding assistance from OpenAI Codex, which helped with implementation, refactoring, tests, documentation, and benchmarking under human direction and review.

Documentation: bakpack.readthedocs.io

Install

The simplest way to install bakpack is to download the latest prebuilt binary from the GitHub releases page:

Choose the archive or binary matching your OS and CPU architecture. Release pages also include a SHA-256 checksum file named like bakpack-v0.1.0-checksums.txt. Use it to verify downloaded archives before installing.

After installing, check the version with:

bakpack --version

If you want to build locally instead:

git clone https://github.com/martinghunt/bakpack.git
cd bakpack
./build.sh

The binary is written to:

build/bakpack

With Go directly:

go install github.com/martinghunt/bakpack/cmd/bakpack@latest

Archive creation uses the command line xz program:

xz -9e -T1 -c

Put xz in PATH. AGC genome input requires agc in PATH.

Quick Start

Make one reduced JSON file:

bakpack reduce sample.bakta.json sample.fa -o sample.reduced.bakta.json

Reconstruct original JSON content:

bakpack restore sample.reduced.bakta.json sample.fa -o sample.bakta.json

Build a compressed archive:

bakpack build \
  --annotations annotations.tar.xz \
  --genomes genomes.tar.xz \
  --output annotations.bakpack

Build from a combined manifest:

bakpack build \
  --manifest samples.tsv \
  --output annotations.bakpack

Extract one reconstructed annotation and its genome FASTA:

bakpack extract annotations.bakpack SAMN1 \
  --genomes genomes.tar.xz \
  --original \
  --genome \
  --output-dir out

The archive path can also be an HTTP(S) URL when the server supports byte-range requests:

bakpack extract https://example.org/annotations.bakpack SAMN1 \
  --genomes genomes.tar.xz \
  --original \
  --output-dir out

Extract several reduced annotations efficiently:

bakpack extract annotations.bakpack SAMN1 SAMN2 SAMN3 \
  --reduced \
  --output-dir out

Documentation

Index

Constants

View Source 
const (
	ArchiveMagic   = "BAKPACK1"
	ArchiveVersion = 1
)

Variables

This section is empty.

Functions

func BuildArchive 

func BuildArchive(ctx context.Context, opts BuildOptions) error

func CanonicalJSON 

func CanonicalJSON(value any) ([]byte, error)

func CanonicalJSONSHA256 

func CanonicalJSONSHA256(value any) (string, error)

func DecodeJSON 

func DecodeJSON(data []byte) (any, error)

func ExtractArchive 

func ExtractArchive(ctx context.Context, opts ExtractOptions) error

func FeatureNT 

func FeatureNT(feature map[string]any, genome Genome) (string, bool)

func JSONBytesCanonicalSHA256 

func JSONBytesCanonicalSHA256(data []byte) (string, error)

func OpenManifestSources 

func OpenManifestSources(path string) (FileSource, FileSource, error)

func PrettyJSON 

func PrettyJSON(value any) ([]byte, error)

func SHA256Hex 

func SHA256Hex(data []byte) string

Types

type AGCGenomeSource 

type AGCGenomeSource struct {
	Path    string
	Command string
	Threads int
}

func (AGCGenomeSource) Get 

func (s AGCGenomeSource) Get(ctx context.Context, sample string) (FileRecord, error)

func (AGCGenomeSource) Order 

func (s AGCGenomeSource) Order(ctx context.Context) ([]string, error)

func (AGCGenomeSource) Records 

func (s AGCGenomeSource) Records(ctx context.Context) ([]FileRecord, error)

type Archive 

type Archive struct {
	// contains filtered or unexported fields
}

Archive is an opened bakpack archive. It keeps the archive index in memory and can extract one or more samples without reopening the archive.

func OpenArchive 

func OpenArchive(ctx context.Context, path string, opts ...OpenArchiveOptions) (*Archive, error)

OpenArchive opens a local or HTTP(S) bakpack archive and reads its front index. HTTP(S) archives are accessed with byte-range requests.

func (*Archive) Close 

func (a *Archive) Close() error

Close closes the underlying archive reader.

func (*Archive) Extract 

func (a *Archive) Extract(ctx context.Context, req ExtractRequest) ([]ExtractedSample, error)

Extract extracts one or more samples from the archive. Without OnSample, results are returned in the same order as req.Samples. With OnSample, results are delivered to the callback and the returned slice is nil.

func (*Archive) Index 

func (a *Archive) Index() ArchiveIndex

Index returns a copy of the archive index.

func (*Archive) SampleIDs 

func (a *Archive) SampleIDs() []string

SampleIDs returns sample IDs in archive order.

type ArchiveIndex 

type ArchiveIndex struct {
	Format        string        `json:"format"`
	Version       int           `json:"version"`
	PayloadFormat string        `json:"payload_format"`
	ChunkSize     int           `json:"chunk_size"`
	Chunks        []ChunkIndex  `json:"chunks"`
	Samples       []SampleIndex `json:"samples"`
}

func ReadArchiveIndex 

func ReadArchiveIndex(path string) (ArchiveIndex, error)

func ReadArchiveIndexContext 

func ReadArchiveIndexContext(ctx context.Context, path string, opts ...OpenArchiveOptions) (ArchiveIndex, error)

ReadArchiveIndexContext reads the front index from a local or HTTP(S) archive. HTTP(S) archives are accessed with byte-range requests.

type BuildOptions 

type BuildOptions struct {
	Annotations FileSource
	Genomes     FileSource
	Order       []string
	ChunkSize   int
	OutputPath  string
	XZThreads   int

	// AnnotationSpoolCompression controls temporary files made when building
	// from annotation tar.xz sources whose order differs from genome order.
	// Supported values are "", "gzip", "none", and "raw"; "" defaults to gzip.
	AnnotationSpoolCompression string
}

type ChunkIndex 

type ChunkIndex struct {
	ID               int                `json:"id"`
	Offset           int64              `json:"offset"`
	CompressedSize   int64              `json:"compressed_size"`
	UncompressedSize int64              `json:"uncompressed_size"`
	TopKeys          []string           `json:"top_keys,omitempty"`
	ValueSchemas     []SchemaIndexEntry `json:"value_schemas,omitempty"`
	FeatureSchemas   []SchemaIndexEntry `json:"feature_schemas,omitempty"`
	FeatureFields    []string           `json:"feature_fields,omitempty"`
	FieldCodecs      []FieldCodec       `json:"field_codecs,omitempty"`
}

type DirSource 

type DirSource struct {
	Dir  string
	Role string
}

func (DirSource) Get 

func (s DirSource) Get(ctx context.Context, sample string) (FileRecord, error)

func (DirSource) Order 

func (s DirSource) Order(ctx context.Context) ([]string, error)

func (DirSource) Records 

func (s DirSource) Records(ctx context.Context) ([]FileRecord, error)

type ExtractOptions 

type ExtractOptions struct {
	ArchivePath string
	Genomes     FileSource
	Samples     []string
	OutputDir   string
	Reduced     bool
	Original    bool
	Genome      bool
}

type ExtractRequest 

type ExtractRequest struct {
	// Genomes is required when Original or Genome output is requested.
	Genomes FileSource
	// Samples are the sample IDs to extract.
	Samples []string
	// Reduced returns reduced Bakta JSON.
	Reduced bool
	// Original reconstructs original Bakta JSON and verifies its canonical hash.
	Original bool
	// Genome returns matching genome FASTA.
	Genome bool
	// OnSample is called for each extracted sample. When nil, Extract returns
	// accumulated results in the same order as Samples.
	OnSample func(ExtractedSample) error
}

ExtractRequest configures extraction from an opened archive.

type ExtractedSample 

type ExtractedSample struct {
	SampleID                    string
	AnnotationName              string
	GenomeName                  string
	ReducedJSON                 []byte
	OriginalJSON                []byte
	GenomeFASTA                 []byte
	OriginalJSONCanonicalSHA256 string
	ReducedJSONCanonicalSHA256  string
}

ExtractedSample contains the bytes extracted or reconstructed for one sample.

type FieldCodec 

type FieldCodec struct {
	Field  string   `json:"field"`
	Kind   string   `json:"kind"`
	Value  any      `json:"value,omitempty"`
	Values []string `json:"values,omitempty"`
}

type FileRecord 

type FileRecord struct {
	SampleID string
	Name     string
	Bytes    []byte
}

type FileSource 

type FileSource interface {
	Records(context.Context) ([]FileRecord, error)
	Get(context.Context, string) (FileRecord, error)
	Order(context.Context) ([]string, error)
}

func OpenSource 

func OpenSource(path, kind, role string) (FileSource, error)

type Genome 

type Genome struct {
	SampleID string
	Filename string
	Contigs  []seqio.SeqRecord
	// contains filtered or unexported fields
}

func ReadGenome 

func ReadGenome(sampleID, filename string, data []byte) (Genome, error)

func (Genome) Contig 

func (g Genome) Contig(name string) ([]byte, bool)

func (Genome) FASTABytes 

func (g Genome) FASTABytes(wrap int) []byte

type JSONChecksums 

type JSONChecksums struct {
	CanonicalSHA256 string `json:"canonical_sha256"`
}

type ListSource 

type ListSource struct {
	Path string
	Role string
}

func (ListSource) Get 

func (s ListSource) Get(ctx context.Context, sample string) (FileRecord, error)

func (ListSource) Order 

func (s ListSource) Order(ctx context.Context) ([]string, error)

func (ListSource) Records 

func (s ListSource) Records(ctx context.Context) ([]FileRecord, error)

type Manifest 

type Manifest struct {
	Path    string
	Entries []ManifestEntry
}

func ReadManifest 

func ReadManifest(path string) (Manifest, error)

type ManifestEntry 

type ManifestEntry struct {
	SampleID       string
	AnnotationPath string
	GenomePath     string
}

type ManifestSource 

type ManifestSource struct {
	Manifest Manifest
	Role     string
}

func (ManifestSource) Get 

func (s ManifestSource) Get(ctx context.Context, sample string) (FileRecord, error)

func (ManifestSource) Order 

func (s ManifestSource) Order(ctx context.Context) ([]string, error)

func (ManifestSource) Records 

func (s ManifestSource) Records(ctx context.Context) ([]FileRecord, error)

type OpenArchiveOptions 

type OpenArchiveOptions struct {
	// HTTPClient is used for byte-range requests when the archive path is an
	// HTTP(S) URL. A nil client uses http.DefaultClient.
	HTTPClient *http.Client
}

OpenArchiveOptions configures archive reads.

type ReduceResult 

type ReduceResult struct {
	ReducedJSON []byte
	Original    JSONChecksums
	Reduced     JSONChecksums
}

func ReduceBaktaJSON 

func ReduceBaktaJSON(original []byte, genome Genome) (ReduceResult, error)

type RestoreResult 

type RestoreResult struct {
	OriginalJSON []byte
	Original     JSONChecksums
}

func RestoreBaktaJSON 

func RestoreBaktaJSON(reduced []byte, genome Genome) (RestoreResult, error)

type SampleIndex 

type SampleIndex struct {
	SampleID                    string `json:"sample_id"`
	AnnotationName              string `json:"annotation_name"`
	GenomeName                  string `json:"genome_name"`
	ChunkID                     int    `json:"chunk_id"`
	OriginalJSONCanonicalSHA256 string `json:"original_json_canonical_sha256"`
	ReducedJSONCanonicalSHA256  string `json:"reduced_json_canonical_sha256"`
}

type SchemaIndexEntry 

type SchemaIndexEntry struct {
	SchemaID int      `json:"schema_id"`
	Keys     []string `json:"keys"`
}

type TarXZSource 

type TarXZSource struct {
	Path string
	Role string
}

func (TarXZSource) Get 

func (s TarXZSource) Get(ctx context.Context, sample string) (FileRecord, error)

func (TarXZSource) Order 

func (s TarXZSource) Order(ctx context.Context) ([]string, error)

func (TarXZSource) Records 

func (s TarXZSource) Records(ctx context.Context) ([]FileRecord, error)

Source Files

View all Source files

Directories

Path Synopsis
cmd
bakpack command
internal
buildinfo

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.