shard

package
v1.4.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 15, 2026 License: MIT Imports: 28 Imported by: 0

Documentation

Overview

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

msgp -file <path to dsort/shard/record_gen.go> -tests=false -marshal=false -unexported Code generated by the command above; see docs/msgp.md. DO NOT EDIT.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

msgp -file <path to dsort/shard/shard.go> -tests=false -marshal=false -unexported Code generated by the command above; see docs/msgp.md. DO NOT EDIT.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2023-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.

Package shard provides Extract(shard), Create(shard), and associated methods across all supported archival formats (see cmn/archive/mime.go)

  • Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	ContentKeyInt    = "int"
	ContentKeyFloat  = "float"
	ContentKeyString = "string"
)
View Source 
const (
	// Extract methods
	ExtractToMem bits = 1 << iota
	ExtractToDisk
	ExtractToWriter
)
View Source 
const (
	// Values are small to save memory.
	OffsetStoreType = "o"
	SGLStoreType    = "s"
	DiskStoreType   = "d"
)

Variables

View Source 
var (
	RWs = map[string]RW{
		archive.ExtTar:    &tarRW{archive.ExtTar},
		archive.ExtTgz:    &tgzRW{archive.ExtTgz},
		archive.ExtTarGz:  &tgzRW{archive.ExtTarGz},
		archive.ExtTarLz4: &tlz4RW{archive.ExtTarLz4},
		archive.ExtZip:    &zipRW{archive.ExtZip},
	}
)

Functions

func IsCompressed 

func IsCompressed(ext string) bool

func ValidateContentKeyTy added in v1.3.22 

func ValidateContentKeyTy(ty string) error

Types

type ContentLoader 

type ContentLoader interface {
	Load(w io.Writer, rec *Record, obj *RecordObj) (int64, error)
}

loads content from local or remote target

type ErrSortingKeyType added in v1.3.22 

type ErrSortingKeyType struct {
	// contains filtered or unexported fields
}

func (*ErrSortingKeyType) Error added in v1.3.22 

func (e *ErrSortingKeyType) Error() string

type ExternalKeyMap added in v1.3.24 

type ExternalKeyMap map[string]struct {
	// contains filtered or unexported fields
}

represents a map where keys are regex patterns and values are associated strings.

func NewExternalKeyMap added in v1.3.24 

func NewExternalKeyMap(n int) ExternalKeyMap

func (ExternalKeyMap) Add added in v1.3.24 

func (ekm ExternalKeyMap) Add(key, value string) error

func (ExternalKeyMap) All added in v1.3.24 

func (ekm ExternalKeyMap) All() (result []string)

func (ExternalKeyMap) Lookup added in v1.3.24 

func (ekm ExternalKeyMap) Lookup(input string) (string, error)

type KeyExtractor 

type KeyExtractor interface {
	PrepareExtractor(name string, r cos.ReadSizer, ext string) (cos.ReadSizer, *SingleKeyExtractor, bool)

	// ExtractKey extracts key from either name or reader (file/sgl)
	ExtractKey(ske *SingleKeyExtractor) (any, error)
}

func NewContentKeyExtractor 

func NewContentKeyExtractor(ty, ext string) (KeyExtractor, error)

func NewMD5KeyExtractor 

func NewMD5KeyExtractor() (KeyExtractor, error)

func NewNameKeyExtractor 

func NewNameKeyExtractor() (KeyExtractor, error)

type RW 

type RW interface {
	Extract(lom *core.LOM, r cos.ReadReaderAt, extractor RecordExtractor, toDisk bool) (int64, int, error)
	Create(s *Shard, w io.Writer, loader ContentLoader) (int64, error)
	IsCompressed() bool
	SupportsOffset() bool
	MetadataSize() int64
}

func NewTarRW 

func NewTarRW() RW

func NewTargzRW 

func NewTargzRW(ext string) RW

func NewTarlz4RW 

func NewTarlz4RW() RW

func NewZipRW 

func NewZipRW() RW

func NopRW 

func NopRW(internal RW) RW

type Record 

type Record struct {
	Key      any    `msg:"k" json:"k"` // Used to determine the sorting order.
	Name     string `msg:"n" json:"n"` // Name that uniquely identifies record across all shards.
	DaemonID string `msg:"d" json:"d"` // ID of the target which maintains the contents for this record.
	// All objects associated with given record. Record can be composed of
	// multiple objects which have the same name but different extension.
	Objects []*RecordObj `msg:"o" json:"o"`
}

Record represents the metadata corresponding to a single file from a shard.

func (*Record) DecodeMsg 

func (z *Record) DecodeMsg(dc *msgp.Reader) (err error)

DecodeMsg implements msgp.Decodable

func (*Record) EncodeMsg 

func (z *Record) EncodeMsg(en *msgp.Writer) (err error)

EncodeMsg implements msgp.Encodable

func (*Record) MakeUniqueName 

func (r *Record) MakeUniqueName(obj *RecordObj) string

func (*Record) Msgsize 

func (z *Record) Msgsize() (s int)

Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message

func (*Record) TotalSize 

func (r *Record) TotalSize() int64

type RecordExtractor 

type RecordExtractor interface {
	RecordWithBuffer(args *extractRecordArgs) (int64, error)
}

type RecordManager 

type RecordManager struct {
	Records *Records
	// contains filtered or unexported fields
}

func NewRecordManager 

func NewRecordManager(bck cmn.Bck, extractCreator RW, keyExtractor KeyExtractor, onDupRecs func(string) error) *RecordManager

func (*RecordManager) Cleanup 

func (recm *RecordManager) Cleanup()

func (*RecordManager) EnqueueRecords 

func (recm *RecordManager) EnqueueRecords(records *Records)

func (*RecordManager) ExtractionPaths 

func (recm *RecordManager) ExtractionPaths() *sync.Map

func (*RecordManager) FreeMem 

func (recm *RecordManager) FreeMem(fullContentPath, newStoreType string, value any, buf []byte) int64

func (*RecordManager) FullContentPath 

func (recm *RecordManager) FullContentPath(obj *RecordObj) string

func (*RecordManager) MergeEnqueuedRecords 

func (recm *RecordManager) MergeEnqueuedRecords()

func (*RecordManager) RecordContents 

func (recm *RecordManager) RecordContents() *sync.Map

func (*RecordManager) RecordWithBuffer 

func (recm *RecordManager) RecordWithBuffer(args *extractRecordArgs) (size int64, err error)

type RecordObj 

type RecordObj struct {
	// Can represent, one of the following:
	//  * Shard name - in case offset is used.
	//  * Key for extractCreator's RecordContents - records stored in SGLs.
	//  * Location (full path) on disk where extracted record has been placed.
	//
	// To get path for given object you need to use `FullContentPath` method.
	ContentPath string `msg:"p"  json:"p"`

	// Filesystem file type where the shard is stored - used to determine
	// location for content path when asking filesystem.
	ObjectFileType string `msg:"ft" json:"ft"`

	// Determines where the record has been stored, can be either: OffsetStoreType,
	// SGLStoreType, DiskStoreType.
	StoreType string `msg:"st" json:"st"`

	// If set, determines the offset in shard file where the record begins.
	Offset       int64  `msg:"f,omitempty" json:"f,string,omitempty"`
	MetadataSize int64  `msg:"ms" json:"ms,string"`
	Size         int64  `msg:"s" json:"s,string"`
	Extension    string `msg:"e" json:"e"`
}

RecordObj describes single object of record. Objects inside a single record have different extensions.

func (*RecordObj) DecodeMsg 

func (z *RecordObj) DecodeMsg(dc *msgp.Reader) (err error)

DecodeMsg implements msgp.Decodable

func (*RecordObj) EncodeMsg 

func (z *RecordObj) EncodeMsg(en *msgp.Writer) (err error)

EncodeMsg implements msgp.Encodable

func (*RecordObj) Msgsize 

func (z *RecordObj) Msgsize() (s int)

Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message

type Records 

type Records struct {
	sync.RWMutex `msg:"-"`
	// contains filtered or unexported fields
}

Records abstract array of records. It safe to be used concurrently.

func NewRecords 

func NewRecords(n int) *Records

NewRecords creates new instance of Records struct and allocates n places for the actual Record's

func (*Records) All 

func (r *Records) All() []*Record

func (*Records) DecodeMsg 

func (z *Records) DecodeMsg(dc *msgp.Reader) (err error)

DecodeMsg implements msgp.Decodable

func (*Records) DeleteDup 

func (r *Records) DeleteDup(name, ext string)

func (*Records) Drain 

func (r *Records) Drain()

func (*Records) EncodeMsg 

func (z *Records) EncodeMsg(en *msgp.Writer) (err error)

EncodeMsg implements msgp.Encodable

func (*Records) Exists 

func (r *Records) Exists(name, ext string) (exists bool)

func (*Records) Find 

func (r *Records) Find(name string) (record *Record, exists bool)

NOTE: must be done under lock

func (*Records) Insert 

func (r *Records) Insert(records ...*Record)

func (*Records) Len 

func (r *Records) Len() int

func (*Records) Less 

func (r *Records) Less(i, j int, keyType string) (bool, error)

func (*Records) MarshalJSON 

func (*Records) MarshalJSON() ([]byte, error)

func (*Records) Msgsize 

func (z *Records) Msgsize() (s int)

Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message

func (*Records) RecordMemorySize 

func (r *Records) RecordMemorySize() (size uint64)

func (*Records) Slice 

func (r *Records) Slice(start, end int) *Records

func (*Records) Swap 

func (r *Records) Swap(i, j int)

func (*Records) TotalObjectCount 

func (r *Records) TotalObjectCount() int

func (*Records) UnmarshalJSON 

func (*Records) UnmarshalJSON([]byte) error

type Shard 

type Shard struct {
	// Size is total size of shard to be created.
	Size int64 `msg:"s"`
	// Records contains all metadata to construct the shard.
	Records *Records `msg:"r"`
	// Name determines the output name of the shard.
	Name string `msg:"n"`
}

Shard represents the metadata required to construct a single shard (aka an archive file).

func (*Shard) DecodeMsg 

func (z *Shard) DecodeMsg(dc *msgp.Reader) (err error)

DecodeMsg implements msgp.Decodable

func (*Shard) EncodeMsg 

func (z *Shard) EncodeMsg(en *msgp.Writer) (err error)

EncodeMsg implements msgp.Encodable

func (*Shard) MarshalJSON 

func (*Shard) MarshalJSON() ([]byte, error)

func (*Shard) Msgsize 

func (z *Shard) Msgsize() (s int)

Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message

func (*Shard) UnmarshalJSON 

func (*Shard) UnmarshalJSON([]byte) error

type SingleKeyExtractor 

type SingleKeyExtractor struct {
	// contains filtered or unexported fields
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.