manifester

Documentation

Index

• Constants
• Variables
• func IsS3NotFound(err error) bool
• func NewManifestServer(cfg Config, mb *ManifestBuilder) (*server, error)
• type BuildArgs
• type ChunkDiffReq
• type ChunkDiffStats
• type ChunkStoreRead
  • func NewChunkStoreReadUrl(url, path string) ChunkStoreRead
• type ChunkStoreWrite
  • func NewChunkStoreWrite(cfg ChunkStoreWriteConfig) (ChunkStoreWrite, error)
• type ChunkStoreWriteConfig
• type Config
• type ManifestBuildRes
• type ManifestBuilder
  • func NewManifestBuilder(cfg ManifestBuilderConfig, cs ChunkStoreWrite) (*ManifestBuilder, error)
  • func (b *ManifestBuilder) Build(ctx context.Context, upstream, storePathHash string, ...) (*ManifestBuildRes, error)
  • func (b *ManifestBuilder) BuildFromNar(ctx context.Context, args *BuildArgs, r io.Reader) (*pb.Manifest, error)
  • func (b *ManifestBuilder) ClearStats()
  • func (b *ManifestBuilder) ManifestAsEntry(ctx context.Context, args *BuildArgs, path string, manifest *pb.Manifest) (*pb.Entry, error)
  • func (b *ManifestBuilder) Stats() Stats
• type ManifestBuilderConfig
• type ManifestReq
  • func (r *ManifestReq) CacheKey() string
• type Stats

Constants

const (
	ChunkDiffMaxDigests = 256
	ChunkDiffMaxBytes   = 16 << 20
)

const (
	DefaultSmallFileCutoff = 224

	SmallManifestCutoff = 32 * 1024
)

Variables

var (
	ErrReq      = errors.New("request err")
	ErrNotFound = errors.New("not found")
	ErrInternal = errors.New("internal err")
)

var (
	// protocol is (mostly) json over http
	ManifestPath  = "/manifest"
	ChunkDiffPath = "/chunkdiff"

	// chunk read protocol
	ChunkReadPath     = "/chunk/"    // digest as final path component
	ManifestCachePath = "/manifest/" // cache key as final path component

	BuildRootPath = "/buildroot/" // written by manifester, read only by gc

	ExpandGz = "gz"
	ExpandXz = "xz"
)

Functions

func IsS3NotFound

func IsS3NotFound(err error) bool

func NewManifestServer

func NewManifestServer(cfg Config, mb *ManifestBuilder) (*server, error)

Types

type BuildArgs

type BuildArgs struct {
	SmallFileCutoff int

	ShardTotal int
	ShardIndex int
	// contains filtered or unexported fields
}

type ChunkDiffReq

type ChunkDiffReq struct {
	Bases []byte
	Reqs  []byte

	// If set: Bases and Reqs each comprise one single file in the given compression
	// format. Pass each one through this decompressor before diffing.
	ExpandBeforeDiff string `json:",omitempty"`
}

TODO: move this to pb so we don't have to use base64?

type ChunkDiffStats

type ChunkDiffStats struct {
	BaseChunks int   `json:"baseC"`
	BaseBytes  int   `json:"baseB"`
	ReqChunks  int   `json:"reqC"`
	ReqBytes   int   `json:"reqB"`
	DiffBytes  int   `json:"diffB"`
	DlTotalMs  int64 `json:"dlMs"`
	ZstdMs     int64 `json:"zstdMs"`
}

Response is compressed concatenation of reqs, using bases as compression base, with ChunkDiffStats (json) appended after that (also compressed). Bases and Reqs do not need to be the same length. (Caller must know the lengths of reqs ahead of time to be able to split the result.) Max number of digests in each is 256, and max size of base data or req data is 16 MiB. (256 * 64 KiB chunks = 16 MiB, larger chunks have lower limit on digests.) Note if running on lambda: after the first 6 MB of streamed data, bandwidth is limited. So aim for responses to be < 6 MB.

ChunkDiffStats must contain only integers! (For now, since we sometimes scan backwards to find the start of the stats. We can relax this requirement if we write a reverse json parser.)

type ChunkStoreRead

type ChunkStoreRead interface {
	// Data will be appended to dst and returned.
	// If dst is given, it must be big enough to hold the full chunk!
	Get(ctx context.Context, key string, dst []byte) ([]byte, error)
}

func NewChunkStoreReadUrl

func NewChunkStoreReadUrl(url, path string) ChunkStoreRead

path should be either ChunkReadPath or ManifestCachePath

type ChunkStoreWrite

type ChunkStoreWrite interface {
	PutIfNotExists(ctx context.Context, path, key string, data []byte) ([]byte, error)
	Get(ctx context.Context, path, key string, dst []byte) ([]byte, error)
}

func NewChunkStoreWrite

func NewChunkStoreWrite(cfg ChunkStoreWriteConfig) (ChunkStoreWrite, error)

type ChunkStoreWriteConfig

type ChunkStoreWriteConfig struct {
	// One of these is required:
	ChunkBucket      string
	ChunkLocalDir    string
	ZstdEncoderLevel int
}

type Config

type Config struct {
	Bind             string
	AllowedUpstreams []string

	ChunkDiffZstdLevel int
	ChunkDiffParallel  int
}

type ManifestBuildRes

type ManifestBuildRes struct {
	CacheKey string // path relative to ManifestCachePath
	Bytes    []byte
}

type ManifestBuilder

type ManifestBuilder struct {
	// contains filtered or unexported fields
}

func NewManifestBuilder

func NewManifestBuilder(cfg ManifestBuilderConfig, cs ChunkStoreWrite) (*ManifestBuilder, error)

func (*ManifestBuilder) Build

func (b *ManifestBuilder) Build(
	ctx context.Context,
	upstream, storePathHash string,
	shardTotal, shardIndex int,
	useLocalStoreDump string,
	writeBuildRoot bool,
) (*ManifestBuildRes, error)

func (*ManifestBuilder) BuildFromNar

func (b *ManifestBuilder) BuildFromNar(ctx context.Context, args *BuildArgs, r io.Reader) (*pb.Manifest, error)

func (*ManifestBuilder) ClearStats

func (b *ManifestBuilder) ClearStats()

func (*ManifestBuilder) ManifestAsEntry

func (b *ManifestBuilder) ManifestAsEntry(ctx context.Context, args *BuildArgs, path string, manifest *pb.Manifest) (*pb.Entry, error)

func (*ManifestBuilder) Stats

func (b *ManifestBuilder) Stats() Stats

If errors occur during building, stats might not be exactly right.

type ManifestBuilderConfig

type ManifestBuilderConfig struct {
	ConcurrentChunkOps int
	ChunkSizer         func(int64) shift.Shift

	// Verify loaded narinfo against these keys. Nil means don't verify.
	PublicKeys []signature.PublicKey
	// Sign manifests with these keys.
	SigningKeys []signature.SecretKey
}

type ManifestReq

type ManifestReq struct {
	Upstream      string
	StorePathHash string

	// TODO: move this to pb and embed a GlobalParams?
	DigestAlgo string
	DigestBits int

	// sharded manifesting (not in cache key, only shard 0 writes to cache)
	ShardTotal int `json:",omitempty"`
	ShardIndex int `json:",omitempty"`
}

func (*ManifestReq) CacheKey

func (r *ManifestReq) CacheKey() string

type Stats

type Stats struct {
	Manifests       int64
	Shards          int64
	TotalChunks     int64
	TotalUncmpBytes int64
	NewChunks       int64
	NewUncmpBytes   int64
	NewCmpBytes     int64
}