Documentation
¶
Index ¶
- Constants
- Variables
- func BlobField(name string, nullable bool) (arrow.Field, error)
- func BlobFieldWithOptions(name string, nullable bool, options BlobFieldOptions) (arrow.Field, error)
- func BuildRQModel(dimension, numBits uint32) (json.RawMessage, error)
- func CommitTransactionBatch(ctx context.Context, uri string, txs []*Transaction, opts ...CommitOption) (*Dataset, *Transaction, error)
- type ApproxMode
- type AutoCleanupParams
- type BasePath
- type BaseStoreParams
- type BatchInfo
- type BatchUDF
- type BatchUDFCheckpointStore
- type BlobDescriptorBuilder
- func (b *BlobDescriptorBuilder) AppendDedicated(blobID uint32, length uint64) error
- func (b *BlobDescriptorBuilder) AppendExternal(baseID uint32, uri string, offset, length uint64) error
- func (b *BlobDescriptorBuilder) AppendInline(data []byte) error
- func (b *BlobDescriptorBuilder) AppendManaged(uri string, offset, length uint64, baseID *uint32) error
- func (b *BlobDescriptorBuilder) AppendNull() error
- func (b *BlobDescriptorBuilder) AppendPacked(blobID uint32, offset, length uint64) error
- func (b *BlobDescriptorBuilder) Close() error
- func (b *BlobDescriptorBuilder) ExtendPacked(blobID uint32, ranges []BlobRange) error
- func (b *BlobDescriptorBuilder) Field() (arrow.Field, error)
- func (b *BlobDescriptorBuilder) Finish() (arrow.Array, arrow.Field, error)
- func (b *BlobDescriptorBuilder) PushExternal(uri string, hasRange bool, offset, length uint64) error
- type BlobFieldOptions
- type BlobFile
- func (f *BlobFile) Close() error
- func (f *BlobFile) Read(p []byte) (int, error)
- func (f *BlobFile) ReadAt(p []byte, off int64) (int, error)
- func (f *BlobFile) ReadRanges(ranges []BlobRange, destinations [][]byte) error
- func (f *BlobFile) Seek(offset int64, whence int) (int64, error)
- func (f *BlobFile) SetReadAhead(size int) error
- func (f *BlobFile) Size() (uint64, error)
- func (f *BlobFile) Tell() (uint64, error)
- type BlobHandling
- type BlobRange
- type BlobRangeRequest
- type BlobReadOptions
- type BlobReadOrder
- type BlobRowSelector
- type Branch
- type BranchIdentifier
- type BranchOption
- type BranchVersionMapping
- type CacheSpec
- type CacheSpecKind
- type CacheStats
- type Code
- type ColumnAlteration
- type ColumnUpdate
- type CommitLease
- type CommitLock
- type CommitLockCallbacks
- type CommitOption
- func WithCommitDataStorageVersion(version string) CommitOption
- func WithCommitDetached(detached bool) CommitOption
- func WithCommitLock(lock *CommitLock) CommitOption
- func WithCommitMaxRetries(retries uint32) CommitOption
- func WithCommitRetryTimeout(timeout time.Duration) CommitOption
- func WithCommitSession(session *Session) CommitOption
- func WithCommitSkipAutoCleanup(skip bool) CommitOption
- func WithCommitStableRowIDs(enabled bool) CommitOption
- func WithCommitStorageOptions(options map[string]string) CommitOption
- func WithCommitStorageOptionsAccessor(accessor *StorageOptionsAccessor) CommitOption
- func WithCommitTimeout(timeout CommitTimeout) CommitOption
- func WithCommitV2ManifestPaths(enabled bool) CommitOption
- type CommitTimeout
- type CommitTimeoutMode
- type CompactionPlan
- type CompactionTask
- func (t *CompactionTask) Execute(ctx context.Context, d *Dataset) (*RewriteResult, error)
- func (t *CompactionTask) FragmentCount() uint64
- func (t *CompactionTask) FragmentMetadataJSON(index uint64) (string, error)
- func (t *CompactionTask) JSON() string
- func (t *CompactionTask) Options() compaction.Options
- func (t *CompactionTask) ReadVersion() uint64
- type CreateIndexOption
- type DataFile
- func (f *DataFile) BaseID() (uint32, bool, error)
- func (f *DataFile) Clone() (*DataFile, error)
- func (f *DataFile) Close() error
- func (f *DataFile) ColumnIndices() ([]int32, error)
- func (f *DataFile) FieldIDs() ([]int32, error)
- func (f *DataFile) FileSize() (uint64, bool, error)
- func (f *DataFile) FileVersion() (uint32, uint32, error)
- func (f *DataFile) Path() (string, error)
- type DataReplacementGroup
- type Dataset
- func Append(ctx context.Context, uri string, reader array.RecordReader, ...) (*Dataset, error)
- func CommitTransaction(ctx context.Context, uri string, tx *Transaction, opts ...CommitOption) (*Dataset, error)
- func CommitTransactionToNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, uri string, ...) (*Dataset, error)
- func Create(ctx context.Context, uri string, reader array.RecordReader, ...) (*Dataset, error)
- func Open(ctx context.Context, uri string, opts ...OpenOption) (*Dataset, error)
- func OpenFromNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, ...) (*Dataset, error)
- func Overwrite(ctx context.Context, uri string, reader array.RecordReader, ...) (*Dataset, error)
- func Write(ctx context.Context, uri string, reader array.RecordReader, ...) (*Dataset, error)
- func WriteToNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, ...) (*Dataset, error)
- func (d *Dataset) AddBases(ctx context.Context, bases []BasePath, transactionProperties map[string]string) (*Dataset, error)
- func (d *Dataset) AddColumnsBatchUDF(ctx context.Context, udf BatchUDF) (*Dataset, error)
- func (d *Dataset) AddColumnsNulls(ctx context.Context, schema *arrow.Schema) (*Dataset, error)
- func (d *Dataset) AddColumnsSQL(ctx context.Context, columns []SQLColumn, batchSize ...uint64) (*Dataset, error)
- func (d *Dataset) AddColumnsStream(ctx context.Context, source array.RecordReader, batchSize ...uint64) (*Dataset, error)
- func (d *Dataset) AlterColumns(ctx context.Context, alterations ...ColumnAlteration) (*Dataset, error)
- func (d *Dataset) BasePaths() (map[uint32]BasePath, error)
- func (d *Dataset) Branch(ctx context.Context, name string) (Branch, error)
- func (d *Dataset) Branches(ctx context.Context) ([]Branch, error)
- func (d *Dataset) CheckoutBranch(ctx context.Context, branch string) (*Dataset, error)
- func (d *Dataset) CheckoutRef(ctx context.Context, ref Ref) (*Dataset, error)
- func (d *Dataset) CheckoutTag(ctx context.Context, tag string) (*Dataset, error)
- func (d *Dataset) CheckoutVersion(ctx context.Context, version uint64) (*Dataset, error)
- func (d *Dataset) Cleanup(ctx context.Context, policy compaction.CleanupPolicy, ...) (compaction.RemovalStats, error)
- func (d *Dataset) CleanupExplain(ctx context.Context, policy compaction.CleanupPolicy, ...) (*compaction.CleanupExplanation, error)
- func (d *Dataset) CleanupOldVersions(ctx context.Context, options compaction.CleanupOldVersionsOptions) (compaction.RemovalStats, error)
- func (d *Dataset) Close() error
- func (d *Dataset) CommitCompaction(ctx context.Context, results []*RewriteResult, options *compaction.Options) (*Dataset, compaction.Metrics, error)
- func (d *Dataset) CommitIndexSegments(ctx context.Context, indexName, column string, segments [][]byte) (*Dataset, error)
- func (d *Dataset) CommitOptimizeIndices(ctx context.Context, results []*IndexSegmentResult, ...) (*Dataset, error)
- func (d *Dataset) CompactFiles(ctx context.Context, options *compaction.Options) (*Dataset, compaction.Metrics, error)
- func (d *Dataset) Config() (map[string]string, error)
- func (d *Dataset) CountRows(ctx context.Context) (uint64, error)
- func (d *Dataset) CountRowsFiltered(ctx context.Context, filter string) (uint64, error)
- func (d *Dataset) CreateBranch(ctx context.Context, name string, ref Ref, opts ...BranchOption) (*Dataset, error)
- func (d *Dataset) CreateDataFile(path string, baseID *uint32) (*DataFile, error)
- func (d *Dataset) CreateFragmentFromFile(ctx context.Context, filename string, fragmentID uint64, physicalRows *uint64) (*Fragment, error)
- func (d *Dataset) CreateScalarIndex(ctx context.Context, column string, indexType index.ScalarType, ...) (*Dataset, error)
- func (d *Dataset) CreateTag(ctx context.Context, name string, ref Ref) error
- func (d *Dataset) CreateVectorIndex(ctx context.Context, column string, options index.VectorOptions, ...) (*Dataset, error)
- func (d *Dataset) CreateVectorIndexWithParams(ctx context.Context, column string, params index.VectorParams, ...) (*Dataset, error)
- func (d *Dataset) CurrentBranchIdentifier(ctx context.Context) (BranchIdentifier, error)
- func (d *Dataset) CurrentVersion(ctx context.Context) (DatasetVersion, error)
- func (d *Dataset) DataStatistics(ctx context.Context) ([]FieldStatistics, error)
- func (d *Dataset) Delete(ctx context.Context, predicate string) (*Dataset, uint64, error)
- func (d *Dataset) DeleteBranch(ctx context.Context, name string) error
- func (d *Dataset) DeleteTag(ctx context.Context, name string) error
- func (d *Dataset) DescribeIndices() ([]index.Description, error)
- func (d *Dataset) DropColumns(ctx context.Context, columns ...string) (*Dataset, error)
- func (d *Dataset) DropIndex(ctx context.Context, name string) (*Dataset, error)
- func (d *Dataset) FieldMetadata(ref FieldReference) (arrow.Metadata, error)
- func (d *Dataset) ForceDeleteBranch(ctx context.Context, name string) error
- func (d *Dataset) FragmentCount() (uint64, error)
- func (d *Dataset) FragmentIDs() ([]uint64, error)
- func (d *Dataset) Fragments() ([]*Fragment, error)
- func (d *Dataset) IndexCount() (uint64, error)
- func (d *Dataset) IndexSegmentUUIDs(name string) ([]index.UUID, error)
- func (d *Dataset) IndexSegments() ([]index.SegmentListing, error)
- func (d *Dataset) IndexStatistics(name string) (index.Statistics, error)
- func (d *Dataset) LatestVersion(ctx context.Context) (uint64, error)
- func (d *Dataset) ListIndices() ([]index.Summary, error)
- func (d *Dataset) Merge(ctx context.Context, leftOn, rightOn string, source array.RecordReader) (*Dataset, error)
- func (d *Dataset) MergeIndexMetadata(ctx context.Context, uuid index.UUID, indexType string, ...) error
- func (d *Dataset) MergeIndexSegments(ctx context.Context, segments [][]byte) ([]byte, error)
- func (d *Dataset) MergeInsert(ctx context.Context, onColumns []string, source array.RecordReader, ...) (*Dataset, MergeInsertResult, error)
- func (d *Dataset) NativeSchema() (*NativeSchema, error)
- func (d *Dataset) NewScalarSegmentBuilder(ctx context.Context, column, name string, indexType index.ScalarType, ...) (*SegmentBuilder, error)
- func (d *Dataset) NewVectorSegmentBuilder(ctx context.Context, column, name string, options index.VectorOptions, ...) (*SegmentBuilder, error)
- func (d *Dataset) OpenFragment(id uint64) (*Fragment, error)
- func (d *Dataset) OptimizeIndices(ctx context.Context, options *compaction.OptimizeOptions) (*Dataset, error)
- func (d *Dataset) PlanCompaction(ctx context.Context, options *compaction.Options) (*CompactionPlan, error)
- func (d *Dataset) PlanOptimizeIndices(ctx context.Context, options *compaction.OptimizeOptions) (*OptimizeIndicesPlan, error)
- func (d *Dataset) PrepareFTSMatchQuery(ctx context.Context, column, query string, options FTSMatchPrepareOptions) (*FTSPreparedQuery, error)
- func (d *Dataset) PrepareFTSPhraseQuery(ctx context.Context, column, query string, options FTSPhrasePrepareOptions) (*FTSPreparedQuery, error)
- func (d *Dataset) PrewarmIndex(ctx context.Context, name string, options index.PrewarmOptions) error
- func (d *Dataset) ReadBlobRanges(ctx context.Context, column string, selector BlobRowSelector, ...) (array.RecordReader, error)
- func (d *Dataset) ReadBlobs(ctx context.Context, column string, selector BlobRowSelector, rows []uint64, ...) (array.RecordReader, error)
- func (d *Dataset) ReadTransaction() (*Transaction, error)
- func (d *Dataset) ReadTransactionByVersion(version uint64) (*Transaction, error)
- func (d *Dataset) ReplaceBranchMetadata(ctx context.Context, name string, metadata map[string]string) error
- func (d *Dataset) ReplaceTagMetadata(ctx context.Context, name string, metadata map[string]string) error
- func (d *Dataset) Restore(ctx context.Context, version uint64) (*Dataset, error)
- func (d *Dataset) Scan(opts ...ScanOption) (*Scanner, error)
- func (d *Dataset) Schema() (*arrow.Schema, error)
- func (d *Dataset) SchemaMetadata() (arrow.Metadata, error)
- func (d *Dataset) TableMetadata() (map[string]string, error)
- func (d *Dataset) Tag(ctx context.Context, name string) (Tag, error)
- func (d *Dataset) Tags(ctx context.Context) ([]Tag, error)
- func (d *Dataset) Take(ctx context.Context, indices []uint64, columns ...string) ([]arrow.RecordBatch, error)
- func (d *Dataset) TakeBlobs(ctx context.Context, rowIDs []uint64, column string) ([]*BlobFile, error)
- func (d *Dataset) TakeBlobsByAddresses(ctx context.Context, rowAddresses []uint64, column string) ([]*BlobFile, error)
- func (d *Dataset) TakeBlobsByIndices(ctx context.Context, indices []uint64, column string) ([]*BlobFile, error)
- func (d *Dataset) TakeRows(ctx context.Context, rowIDs []uint64, columns ...string) ([]arrow.RecordBatch, error)
- func (d *Dataset) TrainIVFModel(ctx context.Context, column string, numPartitions uint32, metric index.Metric, ...) (*Model, error)
- func (d *Dataset) TrainPQModel(ctx context.Context, column string, numSubVectors, numBits uint32, ...) (*Model, error)
- func (d *Dataset) Truncate(ctx context.Context) (*Dataset, error)
- func (d *Dataset) Update(ctx context.Context, predicate string, updates ...ColumnUpdate) (*Dataset, uint64, error)
- func (d *Dataset) UpdateConfig(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
- func (d *Dataset) UpdateFieldMetadata(ctx context.Context, updates ...FieldMetadataUpdate) (*Dataset, error)
- func (d *Dataset) UpdateSchemaMetadata(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
- func (d *Dataset) UpdateTableMetadata(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
- func (d *Dataset) UpdateTag(ctx context.Context, name string, ref Ref) error
- func (d *Dataset) Validate(ctx context.Context) error
- func (d *Dataset) Version() (uint64, error)
- func (d *Dataset) VersionRefs(ctx context.Context) ([]uint64, error)
- func (d *Dataset) Versions(ctx context.Context) ([]DatasetVersion, error)
- type DatasetFileWriterOptions
- type DatasetVersion
- type Error
- type ExternalBlobMode
- type FTSBoostOptions
- type FTSCombinedFieldsOptions
- type FTSCoverage
- type FTSDocumentGranularity
- type FTSMatchOptions
- type FTSMatchPrepareOptions
- type FTSMultiMatchOptions
- type FTSOccur
- type FTSPhraseOptions
- type FTSPhrasePrepareOptions
- type FTSPreparedQuery
- type FTSQuery
- func NewFTSBooleanQuery(occurs []FTSOccur, queries []*FTSQuery) (*FTSQuery, error)
- func NewFTSBoostQuery(positive, negative *FTSQuery, options FTSBoostOptions) (*FTSQuery, error)
- func NewFTSCombinedFieldsQuery(terms string, columns []string, options FTSCombinedFieldsOptions) (*FTSQuery, error)
- func NewFTSMatchQuery(terms, column string, options FTSMatchOptions) (*FTSQuery, error)
- func NewFTSMultiMatchQuery(terms string, columns []string, options FTSMultiMatchOptions) (*FTSQuery, error)
- func NewFTSPhraseQuery(terms, column string, options FTSPhraseOptions) (*FTSQuery, error)
- type FieldMetadataUpdate
- type FieldReference
- type FieldReferenceKind
- type FieldStatistics
- type ForceStorageOptionsProvider
- type Fragment
- func (f *Fragment) AddColumnsBatchUDF(ctx context.Context, udf BatchUDF) (*Fragment, *NativeSchema, error)
- func (f *Fragment) AddColumnsNulls(ctx context.Context, columns *arrow.Schema) (*Fragment, *NativeSchema, error)
- func (f *Fragment) AddColumnsSQL(ctx context.Context, columns []SQLColumn, batchSize ...uint64) (*Fragment, *NativeSchema, error)
- func (f *Fragment) AddColumnsStream(ctx context.Context, source array.RecordReader, batchSize ...uint64) (*Fragment, *NativeSchema, error)
- func (f *Fragment) Close() error
- func (f *Fragment) CountRows(filter *string) (uint64, error)
- func (f *Fragment) DataFileAt(index uint64) (*DataFile, error)
- func (f *Fragment) DataFileCount() (uint64, error)
- func (f *Fragment) DataFileForField(fieldID int32) (*DataFile, bool, error)
- func (f *Fragment) Delete(ctx context.Context, predicate string) (*Fragment, error)
- func (f *Fragment) DeleteRows(ctx context.Context, offsets []uint32) (*Fragment, error)
- func (f *Fragment) DeletedRows() (uint64, error)
- func (f *Fragment) ID() uint64
- func (f *Fragment) IsBound() bool
- func (f *Fragment) LogicalRows() (uint64, error)
- func (f *Fragment) MergeColumns(ctx context.Context, source array.RecordReader, leftOn, rightOn string) (*Fragment, *NativeSchema, error)
- func (f *Fragment) MetadataJSON() (json.RawMessage, error)
- func (f *Fragment) NativeSchema() (*NativeSchema, error)
- func (f *Fragment) PhysicalRows() (uint64, error)
- func (f *Fragment) Scan(opts ...ScanOption) (*Scanner, error)
- func (f *Fragment) Schema() (*arrow.Schema, error)
- func (f *Fragment) String() string
- func (f *Fragment) Take(ctx context.Context, indices []uint32, columns ...string) ([]arrow.RecordBatch, error)
- func (f *Fragment) UpdateColumns(ctx context.Context, source array.RecordReader, leftOn, rightOn string) (*UpdateColumnsResult, error)
- func (f *Fragment) WriteColumnWithVersion(ctx context.Context, schema *NativeSchema, source array.RecordReader, ...) (*DataReplacementGroup, error)
- func (f *Fragment) WriteColumns(ctx context.Context, schema *NativeSchema, source array.RecordReader) (*DataReplacementGroup, error)
- type FragmentCreateOption
- type FragmentFileVersion
- type FragmentOffsets
- type IndexSegmentMetadata
- func (m *IndexSegmentMetadata) BaseID() (uint32, bool, error)
- func (m *IndexSegmentMetadata) Clone() (*IndexSegmentMetadata, error)
- func (m *IndexSegmentMetadata) CloneWithFragmentIDs(fragmentIDs []uint32) (*IndexSegmentMetadata, error)
- func (m *IndexSegmentMetadata) Close() error
- func (m *IndexSegmentMetadata) CoveringFieldIDs() ([]int32, error)
- func (m *IndexSegmentMetadata) CreatedAt() (time.Time, bool, error)
- func (m *IndexSegmentMetadata) DatasetVersion() (uint64, error)
- func (m *IndexSegmentMetadata) Details() (json.RawMessage, bool, error)
- func (m *IndexSegmentMetadata) DetailsTypeURL() (string, bool, error)
- func (m *IndexSegmentMetadata) Encode() ([]byte, error)
- func (m *IndexSegmentMetadata) FieldIDs() ([]int32, error)
- func (m *IndexSegmentMetadata) Files() ([]index.MetadataFile, error)
- func (m *IndexSegmentMetadata) FragmentIDs() ([]uint32, error)
- func (m *IndexSegmentMetadata) IndexVersion() (int, error)
- func (m *IndexSegmentMetadata) Name() (string, error)
- func (m *IndexSegmentMetadata) Type() (index.SegmentIndexType, error)
- func (m *IndexSegmentMetadata) UUID() (index.UUID, error)
- type IndexSegmentResult
- func (r *IndexSegmentResult) Group() uint32
- func (r *IndexSegmentResult) JSON() string
- func (r *IndexSegmentResult) PlanID() index.UUID
- func (r *IndexSegmentResult) ReadVersion() uint64
- func (r *IndexSegmentResult) ReplaceCount() uint64
- func (r *IndexSegmentResult) ReplacedUUID(position uint64) (index.UUID, error)
- func (r *IndexSegmentResult) Segment() (*IndexSegmentMetadata, error)
- type MatchOperator
- type MaterializationStyle
- type MergeInsertParams
- type MergeInsertResult
- type MergeWhenMatched
- type MergeWhenNotMatched
- type MergeWhenNotMatchedBySource
- type MetadataUpdate
- type MetadataUpdateMode
- type Metric
- type Model
- type NativeSchema
- type NullOrder
- type OpenOption
- type Operation
- func NewAppendOperation(fragments ...*Fragment) (*Operation, error)
- func NewDataReplacementOperation(groups ...*DataReplacementGroup) (*Operation, error)
- func NewDeleteOperation(updated []*Fragment, deletedFragmentIDs []uint64, predicate string) (*Operation, error)
- func NewMergeOperation(schema *NativeSchema, fragments []*Fragment, preservesNullability bool) (*Operation, error)
- func NewProjectOperation(schema *NativeSchema, preservesNullability bool) (*Operation, error)
- func NewReserveFragmentsOperation(count uint32) (*Operation, error)
- func NewRestoreOperation(version uint64) (*Operation, error)
- func NewUpdateBasesOperation(bases []BasePath) (*Operation, error)
- func NewUpdateOperation(options UpdateOperationOptions) (*Operation, error)
- func (o *Operation) AppendFragment(index uint64) (*Fragment, error)
- func (o *Operation) AppendFragmentCount() (uint64, error)
- func (o *Operation) AppendFragments() ([]*Fragment, error)
- func (o *Operation) Close() error
- func (o *Operation) DataReplacement(index uint64) (*DataReplacementGroup, error)
- func (o *Operation) DataReplacementCount() (uint64, error)
- func (o *Operation) DataReplacements() ([]*DataReplacementGroup, error)
- func (o *Operation) DeleteDeletedFragmentCount() (uint64, error)
- func (o *Operation) DeleteDeletedFragmentID(index uint64) (uint64, error)
- func (o *Operation) DeleteDeletedFragmentIDs() ([]uint64, error)
- func (o *Operation) DeletePredicate() (string, error)
- func (o *Operation) DeleteUpdatedFragmentCount() (uint64, error)
- func (o *Operation) DeleteUpdatedFragmentMetadata(index uint64) (json.RawMessage, error)
- func (o *Operation) FragmentCount() (uint64, error)
- func (o *Operation) Kind() (OperationKind, error)
- func (o *Operation) MergeFragment(index uint64) (*Fragment, error)
- func (o *Operation) MergeFragmentCount() (uint64, error)
- func (o *Operation) MergeFragments() ([]*Fragment, error)
- func (o *Operation) NativeSchema() (*NativeSchema, error)
- func (o *Operation) PreservesNullability() (bool, error)
- func (o *Operation) ReserveFragmentsCount() (uint32, error)
- func (o *Operation) RestoreVersion() (uint64, error)
- func (o *Operation) Schema() (*arrow.Schema, error)
- func (o *Operation) UpdateBase(index uint64) (BasePath, error)
- func (o *Operation) UpdateBases() ([]BasePath, error)
- func (o *Operation) UpdateBasesCount() (uint64, error)
- func (o *Operation) UpdateFieldsModified() ([]uint32, error)
- func (o *Operation) UpdateUpdatedFragment(index uint64) (*Fragment, error)
- func (o *Operation) UpdateUpdatedFragmentCount() (uint64, error)
- func (o *Operation) UpdateUpdatedFragmentOffsets() (*UpdatedFragmentOffsets, bool, error)
- func (o *Operation) UpdateUpdatedFragments() ([]*Fragment, error)
- type OperationKind
- type OptimizeIndexMerge
- func (m *OptimizeIndexMerge) Execute(ctx context.Context, dataset *Dataset, results []*IndexSegmentResult) (*IndexSegmentResult, error)
- func (m *OptimizeIndexMerge) Group() uint32
- func (m *OptimizeIndexMerge) JSON() string
- func (m *OptimizeIndexMerge) OldSegment(index uint64) (*IndexSegmentMetadata, error)
- func (m *OptimizeIndexMerge) OldSegmentCount() uint64
- func (m *OptimizeIndexMerge) PlanID() index.UUID
- func (m *OptimizeIndexMerge) ReadVersion() uint64
- func (m *OptimizeIndexMerge) ReplaceCount() uint64
- func (m *OptimizeIndexMerge) ReplacedUUID(position uint64) (index.UUID, error)
- type OptimizeIndexTask
- func (t *OptimizeIndexTask) EnablePartitionRebalancing() bool
- func (t *OptimizeIndexTask) Execute(ctx context.Context, dataset *Dataset) (*IndexSegmentResult, error)
- func (t *OptimizeIndexTask) FragmentCount() uint64
- func (t *OptimizeIndexTask) FragmentID(index uint64) (uint32, error)
- func (t *OptimizeIndexTask) Group() uint32
- func (t *OptimizeIndexTask) JSON() string
- func (t *OptimizeIndexTask) NumIndicesToMerge() compaction.Optional[uint64]
- func (t *OptimizeIndexTask) OldSegment(index uint64) (*IndexSegmentMetadata, error)
- func (t *OptimizeIndexTask) OldSegmentCount() uint64
- func (t *OptimizeIndexTask) PlanID() index.UUID
- func (t *OptimizeIndexTask) ReadVersion() uint64
- func (t *OptimizeIndexTask) Reference() (*IndexSegmentMetadata, error)
- func (t *OptimizeIndexTask) Retrain() bool
- type OptimizeIndicesPlan
- func (p *OptimizeIndicesPlan) JSON() string
- func (p *OptimizeIndicesPlan) Merge(index uint64) (*OptimizeIndexMerge, error)
- func (p *OptimizeIndicesPlan) MergeCount() uint64
- func (p *OptimizeIndicesPlan) PlanID() index.UUID
- func (p *OptimizeIndicesPlan) ReadVersion() uint64
- func (p *OptimizeIndicesPlan) Task(index uint64) (*OptimizeIndexTask, error)
- func (p *OptimizeIndicesPlan) TaskCount() uint64
- type OrderDirection
- type Ordering
- type Progress
- type ProgressEvent
- type ProjectionColumn
- type Ref
- type RewriteResult
- func (r *RewriteResult) JSON() string
- func (r *RewriteResult) Metrics() compaction.Metrics
- func (r *RewriteResult) NewFragmentCount() uint64
- func (r *RewriteResult) NewFragmentMetadataJSON(index uint64) (string, error)
- func (r *RewriteResult) OriginalFragmentCount() uint64
- func (r *RewriteResult) OriginalFragmentMetadataJSON(index uint64) (string, error)
- func (r *RewriteResult) ReadVersion() uint64
- type SQLColumn
- type ScanMetric
- type ScanOption
- func WithAdditionalSQLFilter(filter string) ScanOption
- func WithBatchReadahead(batches uint64) ScanOption
- func WithBatchSize(size int64) ScanOption
- func WithBatchSizeBytes(size uint64) ScanOption
- func WithBlobHandling(handling BlobHandling) ScanOption
- func WithColumns(columns ...string) ScanOption
- func WithFTSIndexSegments(uuids ...index.UUID) ScanOption
- func WithFTSQuery(query *FTSQuery) ScanOption
- func WithFTSQueryContext(query *FTSPreparedQuery) ScanOption
- func WithFragmentIDs(fragmentIDs ...uint64) ScanOption
- func WithFragmentReadahead(fragments uint64) ScanOption
- func WithIOBufferSize(size uint64) ScanOption
- func WithIncludeDeletedRows(enabled bool) ScanOption
- func WithLimit(limit int64) ScanOption
- func WithMaterializationStyle(style MaterializationStyle, earlyExcept ...string) ScanOption
- func WithMinHashQuery(column, text string) ScanOption
- func WithNearest(column string, query Vector, k uint32) ScanOption
- func WithNearestBatch(column string, queries Vector, dimension int, k uint32) ScanOption
- func WithNearestMultiVector(column string, vectors Vector, dimension int, k uint32) ScanOption
- func WithOffset(offset int64) ScanOption
- func WithOrderBy(orderings ...Ordering) ScanOption
- func WithProjection(columns ...ProjectionColumn) ScanOption
- func WithRowAddress(enabled bool) ScanOption
- func WithRowID(enabled bool) ScanOption
- func WithRowIDAllowlist(rowIDs ...uint64) ScanOption
- func WithRowIDBlocklist(rowIDs ...uint64) ScanOption
- func WithSQLFilter(filter string) ScanOption
- func WithScalarIndexSegment(segment index.UUID) ScanOption
- func WithScanInOrder(enabled bool) ScanOption
- func WithScanStatistics(callback func(ScanStatistics)) ScanOption
- func WithStrictBatchSize(enabled bool) ScanOption
- func WithSubstraitAggregate(data []byte) ScanOption
- func WithSubstraitFilter(data []byte) ScanOption
- func WithTargetParallelism(tasks uint64) ScanOption
- func WithUseScalarIndex(enabled bool) ScanOption
- func WithUseStats(enabled bool) ScanOption
- func WithVectorApproxMode(mode ApproxMode) ScanOption
- func WithVectorDistanceRange(lower, upper *float32) ScanOption
- func WithVectorEF(ef uint32) ScanOption
- func WithVectorFastSearch(fast bool) ScanOption
- func WithVectorMaxNProbes(n uint32) ScanOption
- func WithVectorMetric(metric VectorMetric) ScanOption
- func WithVectorMinNProbes(n uint32) ScanOption
- func WithVectorNProbes(n uint32) ScanOption
- func WithVectorPrefilter(prefilter bool) ScanOption
- func WithVectorQueryParallelism(n int32) ScanOption
- func WithVectorRefineFactor(factor uint32) ScanOption
- func WithVectorSearchEffort(effort float64) ScanOption
- func WithVectorUseIndex(use bool) ScanOption
- func WithoutScoringAutoprojection() ScanOption
- type ScanStatistics
- type Scanner
- type SegmentBuilder
- type Session
- type StorageOptionsAccessor
- type StorageOptionsProvider
- type Tag
- type Transaction
- func (t *Transaction) Close() error
- func (t *Transaction) Commit(ctx context.Context, uri string, opts ...CommitOption) (*Dataset, error)
- func (t *Transaction) MarshalBinary() ([]byte, error)
- func (t *Transaction) Operation() (*Operation, error)
- func (t *Transaction) Properties() (map[string]string, bool, error)
- func (t *Transaction) ReadVersion() (uint64, error)
- func (t *Transaction) Tag() (string, bool, error)
- func (t *Transaction) UUID() (string, error)
- type TransactionOption
- type UpdateColumnsResult
- type UpdateOperationOptions
- type UpdatedFragmentOffsets
- func (o *UpdatedFragmentOffsets) Clone() (*UpdatedFragmentOffsets, error)
- func (o *UpdatedFragmentOffsets) Close() error
- func (o *UpdatedFragmentOffsets) Count() (uint64, error)
- func (o *UpdatedFragmentOffsets) Entries() ([]FragmentOffsets, error)
- func (o *UpdatedFragmentOffsets) FragmentIDAt(index uint64) (uint64, error)
- func (o *UpdatedFragmentOffsets) OffsetsAt(index uint64) ([]uint32, error)
- type Vector
- type VectorDataType
- type VectorMetric
- type WriteFragmentProgress
- type WriteMode
- type WriteOption
- func WithAutoCleanup(params AutoCleanupParams) WriteOption
- func WithFragmentSchema(schema *NativeSchema) WriteOption
- func WithSkipAutoCleanup(skip bool) WriteOption
- func WithWriteBranch(branch string) WriteOption
- func WithWriteCommitLock(lock *CommitLock) WriteOption
- func WithWriteMode(mode WriteMode) WriteOption
- func WithWriteParams(params WriteParams) WriteOption
- func WithWriteSession(session *Session) WriteOption
- func WithWriteStorageOptions(options map[string]string) WriteOption
- func WithWriteStorageOptionsAccessor(accessor *StorageOptionsAccessor) WriteOption
- func WithWriteTransactionProperties(properties map[string]string) WriteOption
- type WriteParams
- type WriteProgress
- type WriteStats
Constants ¶
const ( MetricL2 = index.MetricL2 MetricCosine = index.MetricCosine MetricDot = index.MetricDot MetricHamming = index.MetricHamming )
const ( // ProgressStart begins a stage. ProgressStart = index.ProgressStart // ProgressAdvance reports completed work within a stage. ProgressAdvance = index.ProgressAdvance // ProgressComplete ends a stage. ProgressComplete = index.ProgressComplete )
Variables ¶
var ( ErrClosed = errors.New("lance: closed") ErrInvalidArgument = errors.New("lance: invalid argument") ErrIO = errors.New("lance: io error") ErrNotFound = errors.New("lance: not found") ErrAlreadyExists = errors.New("lance: already exists") ErrIndex = errors.New("lance: index error") ErrInternal = errors.New("lance: internal error") ErrNotSupported = errors.New("lance: not supported") ErrCommitConflict = errors.New("lance: commit conflict") ErrPanic = errors.New("lance: native panic") )
Sentinel errors classify native and argument failures for errors.Is.
Functions ¶
func BlobField ¶
BlobField returns the native blob v2 field with default storage thresholds. The field is Struct<data: LargeBinary?, uri: Utf8?> with the "lance.blob.v2" extension metadata and belongs in the schema passed to a dataset write.
func BlobFieldWithOptions ¶
func BlobFieldWithOptions(name string, nullable bool, options BlobFieldOptions) (arrow.Field, error)
BlobFieldWithOptions returns the native blob v2 field with explicit thresholds.
func BuildRQModel ¶
func BuildRQModel(dimension, numBits uint32) (json.RawMessage, error)
BuildRQModel returns the portable IVF-RQ rotation artifact for a dimension. The artifact is native JSON and can be passed back through VectorOptions.RabitQModelJSON. A numBits of zero selects the native default; the dimension must be positive and divisible by eight.
func CommitTransactionBatch ¶
func CommitTransactionBatch(ctx context.Context, uri string, txs []*Transaction, opts ...CommitOption) (*Dataset, *Transaction, error)
CommitTransactionBatch publishes append transactions atomically through CommitBuilder::execute_batch.
Types ¶
type ApproxMode ¶
type ApproxMode int32
ApproxMode controls how much accuracy a vector index search trades for speed.
const ( // ApproxModeFast favors speed over accuracy. ApproxModeFast ApproxMode = 0 // ApproxModeNormal balances speed and accuracy. ApproxModeNormal ApproxMode = 1 // ApproxModeAccurate favors accuracy over speed. ApproxModeAccurate ApproxMode = 2 )
type AutoCleanupParams ¶
AutoCleanupParams configures native old-version cleanup.
type BasePath ¶
BasePath is one base-path registration in a dataset's manifest.
ID zero asks Lance to allocate the next unused native base ID. A nil Name passes native None; a non-nil Name, including a pointer to an empty string, passes that name explicitly. IsDatasetRoot marks the dataset root rather than a data-only base.
type BaseStoreParams ¶
type BaseStoreParams struct {
StorageOptionsAccessor *StorageOptionsAccessor
BlockSize *uint64
S3CredentialsRefreshOffset *time.Duration
UseConstantSizeUploadParts *bool
ListIsLexicallyOrdered *bool
}
BaseStoreParams configures representable storage parameters for one base path. Pointer fields preserve native absence versus explicit false or zero.
type BatchUDF ¶
type BatchUDF struct {
Mapper func(arrow.RecordBatch) (arrow.RecordBatch, error)
OutputSchema *arrow.Schema
ReadColumns []string
BatchSize *uint32
Store BatchUDFCheckpointStore
}
BatchUDF describes one mapper used by AddColumnsBatchUDF.
Mapper borrows a Go-owned copy of the native input and returns an owned output reference, which the adapter releases after exporting it. Call Retain before keeping the input or returning an output reference owned elsewhere. Exporting the output retains its buffers without copying them. ReadColumns nil means all columns; a non-nil empty slice means no columns. BatchSize nil leaves the native default; a non-nil value, including zero, is forwarded.
type BatchUDFCheckpointStore ¶
type BatchUDFCheckpointStore interface {
GetBatch(info BatchInfo) (arrow.RecordBatch, bool, error)
InsertBatch(info BatchInfo, batch arrow.RecordBatch) error
GetFragment(fragmentID uint32) (string, bool, error)
InsertFragment(fragmentID uint64, fragmentJSON string) error
}
BatchUDFCheckpointStore persists mapper inputs, outputs and fragment checkpoints during a batch UDF. A non-nil store must implement all methods. A nil Store disables native checkpointing.
GetBatch returns an owned batch reference with found=true. The adapter exports that batch and then releases the reference, so call Retain before returning a cached batch that must stay alive. GetFragment returns found=false when a fragment has no stored checkpoint. InsertBatch receives a borrowed batch; call Retain before keeping it beyond the method. Strings passed to InsertFragment are copied and may be retained.
type BlobDescriptorBuilder ¶
type BlobDescriptorBuilder struct {
// contains filtered or unexported fields
}
BlobDescriptorBuilder builds one native prepared blob descriptor column. The builder is not safe for concurrent use.
Finish consumes the native builder. Later append calls report that it is finished; Field keeps returning the prepared field until Close.
func NewBlobDescriptorBuilder ¶
func NewBlobDescriptorBuilder(column string) (*BlobDescriptorBuilder, error)
NewBlobDescriptorBuilder creates a descriptor builder for column. The prepared output can be written as a blob v2 column.
func (*BlobDescriptorBuilder) AppendDedicated ¶
func (b *BlobDescriptorBuilder) AppendDedicated(blobID uint32, length uint64) error
AppendDedicated appends one dedicated descriptor.
func (*BlobDescriptorBuilder) AppendExternal ¶
func (b *BlobDescriptorBuilder) AppendExternal(baseID uint32, uri string, offset, length uint64) error
AppendExternal appends one external descriptor with an explicit base ID.
func (*BlobDescriptorBuilder) AppendInline ¶
func (b *BlobDescriptorBuilder) AppendInline(data []byte) error
AppendInline appends an owned inline blob value.
func (*BlobDescriptorBuilder) AppendManaged ¶
func (b *BlobDescriptorBuilder) AppendManaged(uri string, offset, length uint64, baseID *uint32) error
AppendManaged appends one managed descriptor. A nil baseID selects native None; a non-nil baseID selects Some, including Some(0). Managed registration and schema opt-in are validated by the dataset write, not by the builder.
func (*BlobDescriptorBuilder) AppendNull ¶
func (b *BlobDescriptorBuilder) AppendNull() error
AppendNull appends a null blob value.
func (*BlobDescriptorBuilder) AppendPacked ¶
func (b *BlobDescriptorBuilder) AppendPacked(blobID uint32, offset, length uint64) error
AppendPacked appends one packed descriptor.
func (*BlobDescriptorBuilder) Close ¶
func (b *BlobDescriptorBuilder) Close() error
Close frees the native builder. It is idempotent.
func (*BlobDescriptorBuilder) ExtendPacked ¶
func (b *BlobDescriptorBuilder) ExtendPacked(blobID uint32, ranges []BlobRange) error
ExtendPacked appends packed descriptors in order. If a later range is invalid, native keeps the already appended prefix and returns an error.
func (*BlobDescriptorBuilder) Field ¶
func (b *BlobDescriptorBuilder) Field() (arrow.Field, error)
Field exports the prepared field. It remains available after Finish.
func (*BlobDescriptorBuilder) Finish ¶
Finish consumes the native builder and returns the owned prepared array and field. The caller owns the array and must release it.
func (*BlobDescriptorBuilder) PushExternal ¶
func (b *BlobDescriptorBuilder) PushExternal(uri string, hasRange bool, offset, length uint64) error
PushExternal appends an external descriptor through native push_external. A false hasRange stores offset 0 and length 0; true forwards the range.
type BlobFieldOptions ¶
BlobFieldOptions selects native blob v2 storage thresholds. A nil threshold keeps the native default. A present zero inline threshold is meaningful; a present zero dedicated threshold is rejected natively.
type BlobFile ¶
type BlobFile struct {
// contains filtered or unexported fields
}
BlobFile is one blob value returned by TakeBlobs. It stays readable after the Dataset that produced it is closed and is safe for concurrent use.
A nil *BlobFile represents a null blob. Reading it reports an error, while Close is a no-op so callers can close every slot uniformly.
BlobFile implements io.Reader, io.ReaderAt, io.Seeker and io.Closer. Reads after the cursor reaches the end return io.EOF; seeking past the end is allowed and a later read returns io.EOF.
func (*BlobFile) Close ¶
Close releases the native blob handle. It is idempotent, waits for in-flight operations, and is a no-op for a nil *BlobFile.
func (*BlobFile) ReadAt ¶
ReadAt reads up to len(p) bytes at off without moving the cursor. When the blob ends inside p, it returns the available prefix with io.EOF. A zero-length read succeeds without checking off against the blob size.
func (*BlobFile) ReadRanges ¶
ReadRanges reads several blob-local byte ranges without moving the cursor. ranges and destinations must have the same length, and each destination must have capacity for its range. Native code validates the complete request before writing any destination, so an error leaves every buffer unchanged.
func (*BlobFile) Seek ¶
Seek moves the cursor and returns the new absolute position. Seeking past the end is allowed; a subsequent read returns io.EOF.
func (*BlobFile) SetReadAhead ¶
SetReadAhead sets the native read-ahead buffer size in bytes. Zero disables read-ahead. The setting does not change visible bytes or the cursor.
type BlobHandling ¶
type BlobHandling int32
BlobHandling controls how blob columns are represented by a scan.
const ( // BlobHandlingDescriptors returns blob columns as descriptors and leaves // other binary columns inline. This is the native default. BlobHandlingDescriptors BlobHandling = 0 // BlobHandlingAllBinary materializes every blob column as binary payloads. BlobHandlingAllBinary BlobHandling = 1 // BlobHandlingAllDescriptions returns every string or binary column with // blob metadata as a descriptor. BlobHandlingAllDescriptions BlobHandling = 2 )
type BlobRangeRequest ¶
BlobRangeRequest asks for a byte range from one selected row. Row is interpreted according to the BlobRowSelector.
type BlobReadOptions ¶
type BlobReadOptions struct {
IOBufferSize *uint64
Order BlobReadOrder
}
BlobReadOptions configures planned blob reads. IOBufferSize is forwarded only when non-nil; an explicit zero is rejected natively. A nil BlobReadOptions selects native defaults.
type BlobReadOrder ¶
type BlobReadOrder int32
BlobReadOrder controls whether planned reads preserve input order.
const ( BlobOrderPreserve BlobReadOrder = 0 BlobOrderUnordered BlobReadOrder = 1 )
type BlobRowSelector ¶
type BlobRowSelector int32
BlobRowSelector selects how read rows are interpreted.
const ( // BlobRowSelectorNone passes no row selection. Native reports an error // when rows or requests are non-empty, and when no selection is supplied. BlobRowSelectorNone BlobRowSelector = 0 // BlobRowSelectorIDs selects by Lance row ID. BlobRowSelectorIDs BlobRowSelector = 1 // BlobRowSelectorAddresses selects by row address. BlobRowSelectorAddresses BlobRowSelector = 2 // BlobRowSelectorIndices selects by zero-based row offset. BlobRowSelectorIndices BlobRowSelector = 3 )
type Branch ¶
type Branch struct {
Name string
ParentBranch string
ParentVersion uint64
CreateAt time.Time
ManifestSize uint64
Metadata map[string]string
Identifier BranchIdentifier
}
Branch is a named, writable line of dataset history.
ParentBranch is empty for a root branch. CreateAt has second precision because the native branch representation stores Unix seconds. Metadata and Identifier are copies and remain valid after the Dataset is closed.
type BranchIdentifier ¶
type BranchIdentifier struct {
VersionMapping []BranchVersionMapping
}
BranchIdentifier maps a branch's versions to the manifest UUIDs they were cloned from.
type BranchOption ¶
type BranchOption func(*branchOptions) error
BranchOption configures CreateBranch.
func WithBranchStorageOptions ¶
func WithBranchStorageOptions(options map[string]string) BranchOption
WithBranchStorageOptions passes storage options to native branch creation.
A nil options map inherits the source dataset's store parameters, matching CreateBranch without this option. A non-nil empty map explicitly passes an empty parameter set, which is distinct from inheriting.
type BranchVersionMapping ¶
BranchVersionMapping records one version-to-manifest mapping in a branch identifier. UUID is empty when the native manifest reference has no UUID.
type CacheSpec ¶
type CacheSpec struct {
Kind CacheSpecKind
SizeBytes uint64
URI string
BackendKind string
Options map[string]string
}
CacheSpec configures an index or metadata cache for NewSessionWithCacheSpecs. Only fields selected by Kind are used.
type CacheSpecKind ¶
type CacheSpecKind int32
CacheSpecKind selects the configuration used by one session cache.
const ( CacheSpecDefault CacheSpecKind = 0 CacheSpecSize CacheSpecKind = 1 CacheSpecURI CacheSpecKind = 2 CacheSpecConfig CacheSpecKind = 3 )
type CacheStats ¶
type CacheStats = sharesession.CacheStats
CacheStats is a snapshot of both session caches.
type ColumnAlteration ¶
ColumnAlteration changes one existing column. Path is required. Rename nil keeps the current name. Nullable nil keeps the current nullability. DataType nil keeps the current type. At least one change must be set.
type ColumnUpdate ¶
ColumnUpdate assigns Value to Column for rows matched by Update.
Value is a SQL scalar expression evaluated by Lance for each row, not a Go value or SQL fragment to be embedded by the caller. For example, "price * 2" doubles the price column and "'unknown'" stores the string unknown.
type CommitLease ¶
type CommitLease any
CommitLease is the opaque value returned by CommitLockCallbacks.Acquire. A nil lease represents stateless coordination.
type CommitLock ¶
type CommitLock struct {
// contains filtered or unexported fields
}
CommitLock owns an external commit lock. Close only drops this handle's owner; datasets that already retained the lock remain usable.
func NewCommitLock ¶
func NewCommitLock(callbacks CommitLockCallbacks) (*CommitLock, error)
NewCommitLock constructs an external commit lock. Construction transfers the callback set only on success; a failed construction invokes no callback.
func (*CommitLock) Close ¶
func (l *CommitLock) Close() error
Close releases the caller's native commit-lock owner. It is idempotent and waits for native calls that already pinned the lock. Callback state must outlive native datasets that retain the lock. Close returns an error only for API symmetry; native close has no failure status.
type CommitLockCallbacks ¶
type CommitLockCallbacks struct {
Acquire func(version uint64) (CommitLease, error)
Release func(lease CommitLease, success bool) error
Destroy func(lease CommitLease)
}
CommitLockCallbacks implements externally coordinated commit serialization.
Acquire and Release run synchronously on native threads. They may be called concurrently but must not re-enter the dataset or transaction currently being committed, or call Close. Acquire may return a nil lease for stateless coordination. If it returns both a lease and an error, native code destroys that partial lease once without calling Release.
Callbacks do not receive a context.Context; native code waits for them to return and cannot cancel an in-flight callback. Release may be called more than once for the same lease; implementations must be idempotent. A release error does not undo a commit that native code has already published. Destroy is optional and runs exactly once for every lease returned by Acquire, including a nil stateless lease.
Callback errors may implement ErrorCode() int32 to select a native error category, including int32(CodeCommitConflict) for the native conflict/retry policy. Wrapped errors retain that classification. Other Go errors use CodeInternal. A conflict classification does not undo a published commit.
type CommitOption ¶
type CommitOption func(*commitConfig) error
CommitOption configures CommitTransaction.
func WithCommitDataStorageVersion ¶
func WithCommitDataStorageVersion(version string) CommitOption
WithCommitDataStorageVersion overrides the data storage version.
func WithCommitDetached ¶
func WithCommitDetached(detached bool) CommitOption
WithCommitDetached writes a detached manifest version without advancing the dataset's latest version.
func WithCommitLock ¶
func WithCommitLock(lock *CommitLock) CommitOption
WithCommitLock routes CommitTransaction and CommitTransactionBatch through an external commit lock.
func WithCommitMaxRetries ¶
func WithCommitMaxRetries(retries uint32) CommitOption
WithCommitMaxRetries sets the commit conflict retry count.
func WithCommitRetryTimeout ¶
func WithCommitRetryTimeout(timeout time.Duration) CommitOption
WithCommitRetryTimeout bounds the total time spent retrying conflict resolution.
func WithCommitSession ¶
func WithCommitSession(session *Session) CommitOption
WithCommitSession pins the commit to a session.
func WithCommitSkipAutoCleanup ¶
func WithCommitSkipAutoCleanup(skip bool) CommitOption
WithCommitSkipAutoCleanup controls native old-version cleanup on commit.
func WithCommitStableRowIDs ¶
func WithCommitStableRowIDs(enabled bool) CommitOption
WithCommitStableRowIDs overrides stable row ID generation.
func WithCommitStorageOptions ¶
func WithCommitStorageOptions(options map[string]string) CommitOption
WithCommitStorageOptions passes object-store options to the commit.
func WithCommitStorageOptionsAccessor ¶
func WithCommitStorageOptionsAccessor(accessor *StorageOptionsAccessor) CommitOption
WithCommitStorageOptionsAccessor supplies a reusable dynamic accessor for a commit. It is mutually exclusive with WithCommitStorageOptions.
func WithCommitTimeout ¶
func WithCommitTimeout(timeout CommitTimeout) CommitOption
WithCommitTimeout sets the native overall commit timeout policy.
func WithCommitV2ManifestPaths ¶
func WithCommitV2ManifestPaths(enabled bool) CommitOption
WithCommitV2ManifestPaths overrides v2 manifest path generation.
type CommitTimeout ¶
type CommitTimeout struct {
Mode CommitTimeoutMode
Duration time.Duration
}
CommitTimeout selects the native overall commit timeout. Duration is used only with CommitTimeoutExplicit.
type CommitTimeoutMode ¶
type CommitTimeoutMode int32
CommitTimeoutMode selects the native overall commit timeout policy.
const ( CommitTimeoutDefault CommitTimeoutMode = 0 CommitTimeoutDisabled CommitTimeoutMode = 1 CommitTimeoutExplicit CommitTimeoutMode = 2 )
type CompactionPlan ¶
type CompactionPlan struct {
// contains filtered or unexported fields
}
CompactionPlan is an immutable snapshot of a native compaction plan. It carries the read version, effective options, task JSON and task snapshots. A plan does not reserve files or publish anything.
func CompactionPlanFromJSON ¶
func CompactionPlanFromJSON(data string) (*CompactionPlan, error)
CompactionPlanFromJSON restores a plan snapshot from native serialized JSON. All task snapshots, effective options and metadata are copied eagerly.
func (*CompactionPlan) JSON ¶
func (p *CompactionPlan) JSON() string
JSON returns the native serialized plan JSON.
func (*CompactionPlan) Options ¶
func (p *CompactionPlan) Options() compaction.Options
Options returns the effective compaction options resolved by the native planner. They include the exact data storage version selected for this plan.
func (*CompactionPlan) ReadVersion ¶
func (p *CompactionPlan) ReadVersion() uint64
ReadVersion returns the dataset version the plan was created from.
func (*CompactionPlan) Task ¶
func (p *CompactionPlan) Task(index uint64) (*CompactionTask, error)
Task returns the task at index. The returned task is an independent snapshot; it can be serialized and executed later.
func (*CompactionPlan) TaskCount ¶
func (p *CompactionPlan) TaskCount() uint64
TaskCount returns the number of compaction tasks in the plan.
type CompactionTask ¶
type CompactionTask struct {
// contains filtered or unexported fields
}
CompactionTask is an immutable compaction task snapshot.
func CompactionTaskFromJSON ¶
func CompactionTaskFromJSON(data string) (*CompactionTask, error)
CompactionTaskFromJSON restores a task snapshot from native serialized JSON.
func (*CompactionTask) Execute ¶
func (t *CompactionTask) Execute(ctx context.Context, d *Dataset) (*RewriteResult, error)
Execute rewrites the task's fragments against d and returns a rewrite result. It writes new data files but does not commit or publish metadata. The result is independent of d; call CommitCompaction to publish it.
The task is executed at its planned read version. If d has advanced since the task was planned, native execution checks out the planned version. ctx is checked before execution starts; it is not observed once native execution is in progress. If execution succeeds after ctx is canceled, the result is returned instead of a cancellation error.
func (*CompactionTask) FragmentCount ¶
func (t *CompactionTask) FragmentCount() uint64
FragmentCount returns the number of source fragments in the task.
func (*CompactionTask) FragmentMetadataJSON ¶
func (t *CompactionTask) FragmentMetadataJSON(index uint64) (string, error)
FragmentMetadataJSON returns one source fragment's metadata JSON.
func (*CompactionTask) JSON ¶
func (t *CompactionTask) JSON() string
JSON returns the native serialized task JSON.
func (*CompactionTask) Options ¶
func (t *CompactionTask) Options() compaction.Options
Options returns the effective options carried by the task.
func (*CompactionTask) ReadVersion ¶
func (t *CompactionTask) ReadVersion() uint64
ReadVersion returns the dataset version the task must execute against.
type CreateIndexOption ¶
type CreateIndexOption func(*createIndexConfig) error
CreateIndexOption configures index creation.
func WithIndexName ¶
func WithIndexName(name string) CreateIndexOption
WithIndexName names the created index. An empty name leaves the index unnamed.
func WithIndexProgress ¶
func WithIndexProgress(progress index.ProgressFunc) CreateIndexOption
WithIndexProgress observes progress while creating an ordinary index. Nil disables observation and preserves the behavior of earlier releases.
The observer runs on native worker threads and may be called concurrently. It must be fast and thread-safe, and it must not call back into Lance, including the Dataset whose index is being built. Native event delivery, stage names and counts are authoritative; observations are advisory and cannot cancel a build. A panic in the observer is recovered and does not stop the build. The observer is retired and in-flight calls are drained before the create call returns, so it is safe to release observer state once that call returns.
func WithIndexReplace ¶
func WithIndexReplace(replace bool) CreateIndexOption
WithIndexReplace replaces an existing index of the same name when true.
type DataFile ¶
type DataFile struct {
// contains filtered or unexported fields
}
DataFile is an owned native data file handle.
func (*DataFile) ColumnIndices ¶
ColumnIndices returns the native column indices stored in the file.
func (*DataFile) FileSize ¶
FileSize returns the data file size and whether native metadata recorded it.
func (*DataFile) FileVersion ¶
FileVersion returns the major and minor data file version.
type DataReplacementGroup ¶
type DataReplacementGroup struct {
// contains filtered or unexported fields
}
DataReplacementGroup is a native fragment data file replacement.
func NewDataReplacementGroup ¶
func NewDataReplacementGroup(fragmentID uint64, newFile *DataFile) (*DataReplacementGroup, error)
NewDataReplacementGroup clones newFile into a replacement group.
func (*DataReplacementGroup) Clone ¶
func (g *DataReplacementGroup) Clone() (*DataReplacementGroup, error)
Clone returns an independent group snapshot.
func (*DataReplacementGroup) Close ¶
func (g *DataReplacementGroup) Close() error
Close releases the group. It is idempotent.
func (*DataReplacementGroup) File ¶
func (g *DataReplacementGroup) File() (*DataFile, error)
File returns the replacement data file.
func (*DataReplacementGroup) FragmentID ¶
func (g *DataReplacementGroup) FragmentID() (uint64, error)
FragmentID returns the replaced fragment ID. Zero is valid.
type Dataset ¶
type Dataset struct {
// contains filtered or unexported fields
}
Dataset is a checked-out snapshot of a Lance dataset. Mutations return a new Dataset and leave this snapshot unchanged. It is safe for concurrent use. Close waits for in-flight operations before releasing the native handle.
func Append ¶
func Append(ctx context.Context, uri string, reader array.RecordReader, opts ...WriteOption) (*Dataset, error)
Append appends rows to a dataset, creating it if the URI does not exist.
func CommitTransaction ¶
func CommitTransaction(ctx context.Context, uri string, tx *Transaction, opts ...CommitOption) (*Dataset, error)
CommitTransaction publishes one transaction through CommitBuilder::execute. The native commit retries conflicts according to the commit options. A canceled context is observed before the native call; if the commit succeeds after cancellation, the committed Dataset is returned.
func CommitTransactionToNamespace ¶
func CommitTransactionToNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, uri string, tx *Transaction, opts ...CommitOption) (*Dataset, error)
CommitTransactionToNamespace publishes tx through a namespace-managed table at uri. A canceled context is observed before the native call; if the commit succeeds after cancellation, the committed Dataset is returned.
func Create ¶
func Create(ctx context.Context, uri string, reader array.RecordReader, opts ...WriteOption) (*Dataset, error)
Create writes a new dataset and fails if the URI already exists.
func OpenFromNamespace ¶
func OpenFromNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, opts ...OpenOption) (*Dataset, error)
OpenFromNamespace opens a table managed by ns. tableID is the native table identifier path. A zero version selects the latest version.
func Overwrite ¶
func Overwrite(ctx context.Context, uri string, reader array.RecordReader, opts ...WriteOption) (*Dataset, error)
Overwrite commits a new version containing only the supplied rows. It creates the dataset if the URI does not exist; earlier versions remain readable.
func Write ¶
func Write(ctx context.Context, uri string, reader array.RecordReader, opts ...WriteOption) (*Dataset, error)
Write writes all batches from reader using the supplied options. The default mode is Create. Auto-cleanup is unset unless WithAutoCleanup is supplied; session and progress options do not change this default. Append and Overwrite preserve the existing dataset's stored cleanup policy.
reader is borrowed and advanced during the call, possibly only partially when a write fails. The caller retains ownership and must release reader after the call. Do not use or release it concurrently with Write.
The native write is synchronous and cannot be interrupted once it starts. A canceled context is observed before the native call; if the write commits after cancellation, the committed Dataset is returned instead of reporting a failure that could be mistaken for an uncommitted write.
func WriteToNamespace ¶
func WriteToNamespace(ctx context.Context, ns namespace.Namespace, tableID []string, reader array.RecordReader, opts ...WriteOption) (*Dataset, error)
WriteToNamespace writes all batches from reader into a table managed by ns. The default mode is Create.
reader is borrowed and advanced during the call. The caller retains ownership and must release it after the call. A canceled context is observed before the native call; if the write commits after cancellation, the committed Dataset is returned.
func (*Dataset) AddBases ¶
func (d *Dataset) AddBases(ctx context.Context, bases []BasePath, transactionProperties map[string]string) (*Dataset, error)
AddBases registers bases in a new version and returns an independent Dataset at that version. d is not changed. Lance validates conflicts and allocates IDs for bases with ID zero; an empty bases slice still commits a version with an unchanged registry.
A nil transactionProperties map passes native None. A non-nil map, including an empty one, passes an explicitly present property map to native code. Native protobuf persistence normalizes empty maps to absent, so ReadTransactionByVersion does not preserve that input distinction.
AddBases records and lists bases; it does not route new writes to a base. Write placement is configured through WriteParams.InitialBases and the TargetBases, TargetBaseNamesOrPaths or TargetAllBases fields.
func (*Dataset) AddColumnsBatchUDF ¶
AddColumnsBatchUDF returns a new Dataset with columns produced by mapper. The operation runs synchronously. Mapper and store callbacks execute while the native dataset lock is held; they must not call back into this Dataset.
func (*Dataset) AddColumnsNulls ¶
AddColumnsNulls returns a new Dataset with the given nullable columns added and filled with nulls. The added schema fields must be top-level and every added field must be nullable.
func (*Dataset) AddColumnsSQL ¶
func (d *Dataset) AddColumnsSQL(ctx context.Context, columns []SQLColumn, batchSize ...uint64) (*Dataset, error)
AddColumnsSQL returns a new Dataset with columns computed by SQL expressions. An optional batchSize controls the backfill batch size; zero uses Lance's default.
func (*Dataset) AddColumnsStream ¶
func (d *Dataset) AddColumnsStream(ctx context.Context, source array.RecordReader, batchSize ...uint64) (*Dataset, error)
AddColumnsStream returns a new Dataset with columns appended from source. source must contain the same number of rows as the dataset and its column names must not collide with existing columns. source is borrowed but may be advanced while the native call runs; release it after this method returns. An optional batchSize controls the backfill batch size; zero uses Lance's default.
func (*Dataset) AlterColumns ¶
func (d *Dataset) AlterColumns(ctx context.Context, alterations ...ColumnAlteration) (*Dataset, error)
AlterColumns returns a new Dataset with the requested column alterations.
func (*Dataset) BasePaths ¶
BasePaths returns the base-path registry of d's snapshot, keyed by native base ID. The returned map and values are independent copies: they stay unchanged when d or another handle commits a new version.
func (*Dataset) Branch ¶
Branch returns one explicit branch by name. The implicit main branch has no catalog entry; use CheckoutBranch("main") to read it.
func (*Dataset) Branches ¶
Branches lists the explicit branches recorded in the shared catalog, ordered by name. The implicit main branch is not included; main is always addressable through BranchRef("main") and CheckoutBranch.
Branches are shared catalog state, not a dataset-local snapshot.
func (*Dataset) CheckoutBranch ¶
CheckoutBranch returns an independent dataset at the latest version of branch. branch must not be empty; "main" is normalized by native code.
func (*Dataset) CheckoutRef ¶
CheckoutRef returns an independent dataset at ref.
func (*Dataset) CheckoutTag ¶
CheckoutTag returns an independent dataset at the version recorded by tag.
func (*Dataset) CheckoutVersion ¶
CheckoutVersion returns an independent dataset at version, resolved on the branch the receiver is checked out on. On a main-branch dataset version is the absolute dataset version.
The returned dataset owns its own native handle and must be closed by the caller. The receiver remains open and unchanged.
func (*Dataset) Cleanup ¶
func (d *Dataset) Cleanup(ctx context.Context, policy compaction.CleanupPolicy, options *compaction.CleanupOperationOptions) (compaction.RemovalStats, error)
Cleanup removes unreferenced storage selected by an explicit policy.
Selectors in policy are independent and combine with AND. Set only one of BeforeVersion or RetainVersions: native cleanup rejects both together. A nil Versions slice means no version selector; a non-nil empty slice is an explicit empty selector that native validation rejects.
options overrides operation settings such as delete concurrency and rate limit. A nil options value uses native operation defaults. When both policy and options set a delete rate limit, the operation option wins.
Cleanup is destructive storage maintenance with no snapshot lease and no rollback after partial deletion. It observes the current dataset state and re-evaluates references when executed. ctx is checked before native cleanup starts; it is not observed while native cleanup is running.
func (*Dataset) CleanupExplain ¶
func (d *Dataset) CleanupExplain(ctx context.Context, policy compaction.CleanupPolicy, options *compaction.CleanupOperationOptions) (*compaction.CleanupExplanation, error)
CleanupExplain reports the storage that the policy would consider removing from the current dataset state.
The explanation is a snapshot, not a deletion plan or a reservation. Files can change before execution, and Cleanup re-evaluates references. MaxCandidateFiles limits only the reported candidate list; aggregate stats and the truncation flag describe the full candidate set.
CleanupExplain deletes no files and does not take a snapshot lease. ctx is checked before native work starts; it is not observed while native work is running.
func (*Dataset) CleanupOldVersions ¶
func (d *Dataset) CleanupOldVersions(ctx context.Context, options compaction.CleanupOldVersionsOptions) (compaction.RemovalStats, error)
CleanupOldVersions removes old versions and their unreferenced files.
Cleanup is destructive storage maintenance. It does not take a snapshot lease, does not change the dataset's current version, and cannot roll back files that were already deleted when a later deletion fails. Snapshot handles do not protect removed files: a Dataset or scanner opened at an old version may lose access to its storage after cleanup.
OlderThan is relative to the current time and is evaluated by native Lance. The native operation is synchronous. ctx is checked before work starts; it is not observed once native cleanup is in progress. If cleanup succeeds after ctx is canceled, its removal stats are returned instead of a cancellation error.
func (*Dataset) Close ¶
Close releases the dataset. It is idempotent and waits for active operations before calling native close.
func (*Dataset) CommitCompaction ¶
func (d *Dataset) CommitCompaction(ctx context.Context, results []*RewriteResult, options *compaction.Options) (*Dataset, compaction.Metrics, error)
CommitCompaction commits rewrite results and returns a new Dataset at the committed version. d is not changed: its version, rows and open scanners keep the pre-commit snapshot.
Results may come from Execute or RewriteResultFromJSON. A nil options value asks native commit to use its default options. Pass the plan or task Options to use the exact settings that produced the results.
The native commit is synchronous. ctx is checked before work starts; it is not observed once the commit is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned.
func (*Dataset) CommitIndexSegments ¶
func (d *Dataset) CommitIndexSegments(ctx context.Context, indexName, column string, segments [][]byte) (*Dataset, error)
CommitIndexSegments commits assembled segment metadata as one new dataset version and returns the dataset at that version. The receiver keeps its snapshot. Native validation owns segment compatibility, coverage, and replacement rules, so an invalid set is rejected without changing a version.
func (*Dataset) CommitOptimizeIndices ¶
func (d *Dataset) CommitOptimizeIndices(ctx context.Context, results []*IndexSegmentResult, options *compaction.OptimizeOptions) (*Dataset, error)
CommitOptimizeIndices commits optimized index results and returns a new Dataset at the committed version. d is not changed: its version, rows and open scanners keep the pre-commit snapshot.
Results may come from task or merge Execute, or from IndexSegmentResultFromJSON. Native commit semantics are preserved: empty results retain the native no-op/MemWAL behavior, and every result must have been planned from the same dataset version.
Native plan, task, merge and result JSON does not serialize OptimizeOptions. Pass TransactionProperties again when the results crossed a JSON transport boundary.
The native commit is synchronous. ctx is checked before work starts; it is not observed once the commit is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned.
func (*Dataset) CompactFiles ¶
func (d *Dataset) CompactFiles(ctx context.Context, options *compaction.Options) (*Dataset, compaction.Metrics, error)
CompactFiles rewrites small fragments in d and returns a new Dataset at the committed compaction version. Native compaction may commit fragment-ID reservations before the rewrite, so the version need not advance by one. d is not changed: its version, rows and open scanners keep the pre-compaction snapshot.
A nil options value asks native compaction to derive its effective options from the dataset configuration. A non-nil options value starts from native defaults and applies the fields that are set.
Compaction writes new data files before committing metadata. The native operation is synchronous. ctx is checked before work starts; it is not observed once the native commit is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned instead of an error that could be mistaken for an uncommitted compaction.
func (*Dataset) CountRowsFiltered ¶
CountRowsFiltered returns the row count matching a SQL filter.
func (*Dataset) CreateBranch ¶
func (d *Dataset) CreateBranch(ctx context.Context, name string, ref Ref, opts ...BranchOption) (*Dataset, error)
CreateBranch creates branch name from ref and returns an independent dataset checked out on the new branch.
The returned dataset must be closed by the caller. The receiver remains open and unchanged. Branches are shared catalog state; other open datasets observe the new branch through their next catalog read.
func (*Dataset) CreateDataFile ¶
CreateDataFile creates a native data file handle at path.
func (*Dataset) CreateFragmentFromFile ¶
func (d *Dataset) CreateFragmentFromFile(ctx context.Context, filename string, fragmentID uint64, physicalRows *uint64) (*Fragment, error)
CreateFragmentFromFile creates one detached fragment from filename, a Lance data file stored under the dataset's data directory. filename is relative to that directory; it is not a standalone absolute path to an arbitrary file.
The dataset supplies the snapshot, schema and storage options used to read the file. physicalRows is nil to let native code infer the row count. A non-nil pointer is forwarded explicitly, including a pointer to zero. Native treats this as a hint for legacy files; V2 files use their stored row count.
Native code rejects files written with the Blob v2 layout because their stored schema cannot be projected without the descriptor columns; that limitation is native behavior and is not worked around here.
The returned fragment is uncommitted metadata. Pass it to NewAppendOperation and commit the transaction to publish it.
func (*Dataset) CreateScalarIndex ¶
func (d *Dataset) CreateScalarIndex(ctx context.Context, column string, indexType index.ScalarType, params json.RawMessage, options index.ScalarOptions, opts ...CreateIndexOption) (*Dataset, error)
CreateScalarIndex builds a scalar index and returns a new Dataset snapshot. params is forwarded verbatim as the native plugin JSON; nil leaves the native parameter string unset. Unknown plugin keys are preserved. WithIndexProgress observes the build. An error can occur after the version is committed, during metadata reload; reopen the dataset to inspect the published state before retrying.
func (*Dataset) CreateTag ¶
CreateTag creates a tag pointing at ref.
Tags are shared catalog state: other open datasets observe the tag through their next catalog read, not through a dataset-local snapshot.
func (*Dataset) CreateVectorIndex ¶
func (d *Dataset) CreateVectorIndex(ctx context.Context, column string, options index.VectorOptions, opts ...CreateIndexOption) (*Dataset, error)
CreateVectorIndex builds a vector index and returns a new Dataset snapshot. This Dataset keeps its current version and catalog. Options cover every native vector index build field; nil optional fields keep native defaults. WithIndexProgress observes the build. An error can occur after the index version is committed, during metadata reload. Reopen the dataset to determine the published state before retrying.
func (*Dataset) CreateVectorIndexWithParams ¶
func (d *Dataset) CreateVectorIndexWithParams(ctx context.Context, column string, params index.VectorParams, opts ...CreateIndexOption) (*Dataset, error)
CreateVectorIndexWithParams builds a vector index from legacy parameters. It is equivalent to CreateVectorIndex with params.Options(). WithIndexProgress observes the build.
func (*Dataset) CurrentBranchIdentifier ¶
func (d *Dataset) CurrentBranchIdentifier(ctx context.Context) (BranchIdentifier, error)
CurrentBranchIdentifier reports the lineage of the receiver's branch.
func (*Dataset) CurrentVersion ¶
func (d *Dataset) CurrentVersion(ctx context.Context) (DatasetVersion, error)
CurrentVersion returns the version checked out by this dataset.
func (*Dataset) DataStatistics ¶
func (d *Dataset) DataStatistics(ctx context.Context) ([]FieldStatistics, error)
DataStatistics computes per-field on-disk statistics.
func (*Dataset) Delete ¶
Delete returns a new Dataset at a committed version with the rows matching predicate removed, along with the number of deleted rows. d is not changed: its version, rows and open scanners keep the pre-delete snapshot.
The returned Dataset owns an independent native handle and must be closed by the caller. Closing either dataset does not affect the other.
The native delete is synchronous. ctx is checked before the operation starts; it is not observed once the native commit is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned instead of an error that could be mistaken for an uncommitted mutation.
func (*Dataset) DeleteBranch ¶
DeleteBranch removes a branch using native reference-protection rules.
Deletion fails when child branches or tags still reference the branch.
func (*Dataset) DescribeIndices ¶
func (d *Dataset) DescribeIndices() ([]index.Description, error)
DescribeIndices describes every index, including system indexes.
func (*Dataset) DropColumns ¶
DropColumns returns a new Dataset with the named columns removed.
func (*Dataset) FieldMetadata ¶
func (d *Dataset) FieldMetadata(ref FieldReference) (arrow.Metadata, error)
FieldMetadata returns the metadata of a field addressed by id or nested path.
func (*Dataset) ForceDeleteBranch ¶
ForceDeleteBranch removes a branch even when its native branch contents are missing, for recovery from an interrupted branch creation. Child branches and tags still protect the branch from deletion.
func (*Dataset) FragmentCount ¶
FragmentCount returns the number of fragments in the current dataset version.
func (*Dataset) FragmentIDs ¶
FragmentIDs returns the fragment IDs in manifest order.
func (*Dataset) IndexCount ¶
IndexCount returns the number of user indexes in this snapshot.
func (*Dataset) IndexSegmentUUIDs ¶
IndexSegmentUUIDs returns the raw UUIDs of the physical segments of one index.
func (*Dataset) IndexSegments ¶
func (d *Dataset) IndexSegments() ([]index.SegmentListing, error)
IndexSegments lists the physical segments of all user indexes.
func (*Dataset) IndexStatistics ¶
func (d *Dataset) IndexStatistics(name string) (index.Statistics, error)
IndexStatistics returns the family-specific statistics JSON for one index.
func (*Dataset) LatestVersion ¶
LatestVersion returns the latest committed dataset version.
func (*Dataset) ListIndices ¶
ListIndices lists user indexes. System indexes are omitted.
func (*Dataset) Merge ¶
func (d *Dataset) Merge(ctx context.Context, leftOn, rightOn string, source array.RecordReader) (*Dataset, error)
Merge returns a new Dataset at a committed version after left-joining the non-key columns of source into d. d is not changed.
leftOn may name a dataset column, _rowid or _rowaddr. rightOn names the join field in source. Field lookup is exact and case-sensitive. Empty key strings are forwarded to native validation. The source field named by rightOn is the join key and is not appended; every other source field must be new to d.
source is borrowed and may be partially or fully advanced, including on failure. The caller retains ownership and must release source after this method returns. Do not use or release it concurrently with the call. Argument validation failures before stream export leave source untouched.
The returned Dataset owns an independent native handle and must be closed by the caller. The native merge is synchronous; ctx is checked before the operation starts and is not observed once the native commit is in progress.
func (*Dataset) MergeIndexMetadata ¶
func (d *Dataset) MergeIndexMetadata(ctx context.Context, uuid index.UUID, indexType string, options index.MergeMetadataOptions) error
MergeIndexMetadata writes and merges an index's root artifact without committing a dataset version. At this native revision only the inverted root artifact is mergeable; other index types fail with a native error. A failure may leave partially written files behind, so retrying the same merge is the intended recovery. Callers publish the result with their own index creation path once every worker has merged its portion.
func (*Dataset) MergeIndexSegments ¶
MergeIndexSegments merges uncommitted segment metadata into one uncommitted metadata value. Native validation owns compatibility, replay, staleness, and coverage. The receiver snapshot is not modified and the merged metadata still has to be committed.
func (*Dataset) MergeInsert ¶
func (d *Dataset) MergeInsert(ctx context.Context, onColumns []string, source array.RecordReader, params MergeInsertParams) (*Dataset, MergeInsertResult, error)
MergeInsert returns a new Dataset at a committed version after merging source into d on onColumns. d is not changed.
params controls matched and unmatched row behavior. Its zero value performs find-or-create: existing keys are kept and missing source keys are inserted. See MergeInsertParams for expression requirements.
source is borrowed and may be partially or fully advanced, including on failure. Its schema must be compatible with d. The caller retains ownership and must release source after this method returns. Do not use or release it concurrently with the call. Argument validation failures before stream export leave source untouched.
The returned Dataset owns an independent native handle and must be closed by the caller. The native merge is synchronous; ctx is checked before the operation starts and is not observed once the native commit is in progress.
func (*Dataset) NativeSchema ¶
func (d *Dataset) NativeSchema() (*NativeSchema, error)
NativeSchema clones the dataset's native schema. The schema preserves field IDs, nested metadata and encodings that an Arrow schema view cannot.
func (*Dataset) NewScalarSegmentBuilder ¶
func (d *Dataset) NewScalarSegmentBuilder(ctx context.Context, column, name string, indexType index.ScalarType, params json.RawMessage, options index.ScalarOptions, scope index.SegmentScope) (*SegmentBuilder, error)
NewScalarSegmentBuilder prepares a scalar index segment such as an inverted or bitmap segment for the current dataset snapshot.
func (*Dataset) NewVectorSegmentBuilder ¶
func (d *Dataset) NewVectorSegmentBuilder(ctx context.Context, column, name string, options index.VectorOptions, scope index.SegmentScope) (*SegmentBuilder, error)
NewVectorSegmentBuilder prepares a vector index segment for the current dataset snapshot. options describe the final index (type, metric, and training inputs); scope restricts the segment to a fragment subset and optionally assigns its index UUID.
The builder reuses trained IVF centroids and PQ codebooks carried by options and trains any missing model locally; options.Train defaults to true, matching the native default. Set Train false to require the complete supplied model set and skip training. Training with supplied PQ codebooks and the dot metric requires the scope to cover every fragment, while Train false accepts fragment subsets at this native revision.
func (*Dataset) OpenFragment ¶
OpenFragment opens one fragment by ID.
func (*Dataset) OptimizeIndices ¶
func (d *Dataset) OptimizeIndices(ctx context.Context, options *compaction.OptimizeOptions) (*Dataset, error)
OptimizeIndices runs native single-process index optimization and returns a new Dataset at the committed version. Native optimization plans, executes and merges index segments in turn before committing the merged results. d is not changed: its version, rows and open scanners keep the pre-optimization snapshot.
A nil options value selects native defaults. A non-nil value starts from native defaults and applies the fields that are set.
Native optimization is synchronous. ctx is checked before work starts; it is not observed once native work is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned instead of an error that could be mistaken for an uncommitted optimization. A clean dataset is a native no-op and leaves the version unchanged.
func (*Dataset) PlanCompaction ¶
func (d *Dataset) PlanCompaction(ctx context.Context, options *compaction.Options) (*CompactionPlan, error)
PlanCompaction creates a compaction plan from the current dataset snapshot. It writes no files and publishes no version. options follows the same rules as CompactFiles.
func (*Dataset) PlanOptimizeIndices ¶
func (d *Dataset) PlanOptimizeIndices(ctx context.Context, options *compaction.OptimizeOptions) (*OptimizeIndicesPlan, error)
PlanOptimizeIndices creates an index-optimization plan from the current dataset snapshot. Planning may write model files but publishes no version. options follows the same rules as OptimizeIndices.
func (*Dataset) PrepareFTSMatchQuery ¶
func (d *Dataset) PrepareFTSMatchQuery(ctx context.Context, column, query string, options FTSMatchPrepareOptions) (*FTSPreparedQuery, error)
PrepareFTSMatchQuery plans a Match query against this exact snapshot. The prepared value remains usable after this Dataset is closed; Close waits for in-flight scanner attachment.
func (*Dataset) PrepareFTSPhraseQuery ¶
func (d *Dataset) PrepareFTSPhraseQuery(ctx context.Context, column, query string, options FTSPhrasePrepareOptions) (*FTSPreparedQuery, error)
PrepareFTSPhraseQuery plans a Phrase query against this exact snapshot.
func (*Dataset) PrewarmIndex ¶
func (d *Dataset) PrewarmIndex(ctx context.Context, name string, options index.PrewarmOptions) error
PrewarmIndex warms the named index into the configured session cache. A nil SegmentUUIDs selection warms every segment; a non-nil empty selection warms none. Prewarming is best-effort cache population, not a residency guarantee.
func (*Dataset) ReadBlobRanges ¶
func (d *Dataset) ReadBlobRanges(ctx context.Context, column string, selector BlobRowSelector, requests []BlobRangeRequest, options *BlobReadOptions) (array.RecordReader, error)
ReadBlobRanges plans per-row blob-range reads and returns an incremental Arrow record reader. The schema is request_index: UInt64 (non-null), row_address: UInt64 (non-null), offset: UInt64 (non-null), length: UInt64 (non-null), data: LargeBinary (nullable). request_index preserves the zero-based input position for duplicate and unordered requests.
The context is checked before planning and between batches. Native planning and payload I/O cannot be interrupted once a call is in flight.
func (*Dataset) ReadBlobs ¶
func (d *Dataset) ReadBlobs(ctx context.Context, column string, selector BlobRowSelector, rows []uint64, options *BlobReadOptions) (array.RecordReader, error)
ReadBlobs plans a whole-blob read and returns an incremental Arrow record reader. The schema is row_address: UInt64 (non-null), data: LargeBinary (nullable). A null blob produces a null data value; an empty blob produces a non-null empty value. Batches own their buffers and stay valid after the reader is released and after the dataset is closed.
The context is checked before planning and between batches. Native planning and payload I/O cannot be interrupted once a call is in flight.
func (*Dataset) ReadTransaction ¶
func (d *Dataset) ReadTransaction() (*Transaction, error)
ReadTransaction returns the current snapshot's transaction, or (nil, nil) when the manifest has no transaction file.
func (*Dataset) ReadTransactionByVersion ¶
func (d *Dataset) ReadTransactionByVersion(version uint64) (*Transaction, error)
ReadTransactionByVersion reads a specific version's transaction. It returns (nil, nil) when the version exists but has no transaction file. Missing or invalid versions follow the same native errors as other versioned reads.
func (*Dataset) ReplaceBranchMetadata ¶
func (d *Dataset) ReplaceBranchMetadata(ctx context.Context, name string, metadata map[string]string) error
ReplaceBranchMetadata replaces all metadata on an existing branch. An empty map removes every metadata entry.
func (*Dataset) ReplaceTagMetadata ¶
func (d *Dataset) ReplaceTagMetadata(ctx context.Context, name string, metadata map[string]string) error
ReplaceTagMetadata replaces all metadata on an existing tag. An empty map removes every metadata entry.
func (*Dataset) Restore ¶
Restore commits a new dataset version whose content is version and returns an independent dataset at that new commit. It never rewinds the receiver.
Restore always commits, including when version is already the latest version. The committed version is reported as success even if ctx is canceled while the synchronous native restore is running, because the new commit cannot be rolled back safely. version must be greater than zero.
func (*Dataset) Scan ¶
func (d *Dataset) Scan(opts ...ScanOption) (*Scanner, error)
Scan creates a scanner for the dataset.
func (*Dataset) SchemaMetadata ¶
SchemaMetadata returns the dataset schema's metadata.
func (*Dataset) TableMetadata ¶
TableMetadata returns the dataset's table metadata.
func (*Dataset) Take ¶
func (d *Dataset) Take(ctx context.Context, indices []uint64, columns ...string) ([]arrow.RecordBatch, error)
Take reads rows by zero-based offsets. The caller owns the returned batches.
func (*Dataset) TakeBlobs ¶
func (d *Dataset) TakeBlobs(ctx context.Context, rowIDs []uint64, column string) ([]*BlobFile, error)
TakeBlobs reads blob handles by Lance row ID, preserving input order and duplicates. A NULL native slot becomes a nil *BlobFile. The caller owns the returned handles and must close each non-nil handle.
func (*Dataset) TakeBlobsByAddresses ¶
func (d *Dataset) TakeBlobsByAddresses(ctx context.Context, rowAddresses []uint64, column string) ([]*BlobFile, error)
TakeBlobsByAddresses reads blob handles by row address, preserving input order and duplicates. The caller owns the returned handles.
func (*Dataset) TakeBlobsByIndices ¶
func (d *Dataset) TakeBlobsByIndices(ctx context.Context, indices []uint64, column string) ([]*BlobFile, error)
TakeBlobsByIndices reads blob handles by zero-based row offset, preserving input order and duplicates. The caller owns the returned handles.
func (*Dataset) TakeRows ¶
func (d *Dataset) TakeRows(ctx context.Context, rowIDs []uint64, columns ...string) ([]arrow.RecordBatch, error)
TakeRows reads rows by Lance row ID. Missing or deleted IDs may be omitted. The caller owns the returned batches.
func (*Dataset) TrainIVFModel ¶
func (d *Dataset) TrainIVFModel(ctx context.Context, column string, numPartitions uint32, metric index.Metric, fragmentIDs []uint32, options index.TrainerOptions) (*Model, error)
TrainIVFModel trains IVF centroids on the current dataset snapshot and returns them as a Model wrapping a FixedSizeList<Float32>[dimension] array.
A nil fragmentIDs trains on the whole dataset. A non-nil slice restricts training to those fragments and must not be empty; native validation rejects unknown or duplicate fragment IDs and owns the remaining parameter checks, including metric compatibility and a positive partition count.
func (*Dataset) TrainPQModel ¶
func (d *Dataset) TrainPQModel(ctx context.Context, column string, numSubVectors, numBits uint32, metric index.Metric, ivfCentroids arrow.Array, fragmentIDs []uint32, options index.TrainerOptions) (*Model, error)
TrainPQModel trains a product-quantization codebook conditioned on IVF centroids. The returned Model wraps a FixedSizeList<Float32> array holding numSubVectors*2^numBits rows of dimension/numSubVectors values, matching the native trainer.
The centroids array is borrowed only for this call and remains owned by the caller. Native training reads the model provenance stored in the centroids field metadata, so centroids must carry it; the Model returned by TrainIVFModel does. Centroids without provenance, or trained with a different metric or dimension, are rejected. fragmentIDs follows the same contract as TrainIVFModel.
func (*Dataset) Truncate ¶
Truncate returns a new Dataset at a committed version with no rows. d is not changed, and its historical versions remain readable.
The returned Dataset owns an independent native handle and must be closed by the caller. The native truncate is synchronous; ctx is checked before the operation starts and is not observed once the native commit is in progress.
func (*Dataset) Update ¶
func (d *Dataset) Update(ctx context.Context, predicate string, updates ...ColumnUpdate) (*Dataset, uint64, error)
Update returns a new Dataset at a committed version with updates applied to rows matching predicate, along with the number of updated rows. An empty predicate updates every row. d is not changed.
Each ColumnUpdate.Value is a SQL scalar expression evaluated by Lance per row. At least one update is required. The returned Dataset owns an independent native handle and must be closed by the caller.
The native update is synchronous. ctx is checked before the operation starts; it is not observed once the native commit is in progress. If the commit succeeds after ctx is canceled, the committed Dataset is returned.
func (*Dataset) UpdateConfig ¶
func (d *Dataset) UpdateConfig(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
UpdateConfig returns a new Dataset with configuration updated according to mode. MetadataMerge changes only supplied keys; MetadataReplace replaces the complete configuration. A nil value removes a key under either mode. The returned map is the complete resulting configuration.
func (*Dataset) UpdateFieldMetadata ¶
func (d *Dataset) UpdateFieldMetadata(ctx context.Context, updates ...FieldMetadataUpdate) (*Dataset, error)
UpdateFieldMetadata returns a new Dataset with metadata updated on one or more fields. Each update may merge or replace that field's metadata.
func (*Dataset) UpdateSchemaMetadata ¶
func (d *Dataset) UpdateSchemaMetadata(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
UpdateSchemaMetadata returns a new Dataset with schema metadata updated. Field metadata is preserved; only the schema-level metadata map changes.
func (*Dataset) UpdateTableMetadata ¶
func (d *Dataset) UpdateTableMetadata(ctx context.Context, mode MetadataUpdateMode, updates ...MetadataUpdate) (*Dataset, map[string]string, error)
UpdateTableMetadata returns a new Dataset with table metadata updated. MetadataMerge updates only the supplied keys; MetadataReplace replaces the complete table metadata map. A nil value removes a key.
func (*Dataset) UpdateTag ¶
UpdateTag points an existing tag at ref and preserves its creation time.
func (*Dataset) VersionRefs ¶
VersionRefs returns the lightweight attached-version manifest reference IDs, ordered by version. These are storage references, not full version snapshots.
type DatasetFileWriterOptions ¶
type DatasetFileWriterOptions struct {
DataCacheBytes *uint64
MaxPageBytes *uint64
KeepOriginalArray *bool
}
DatasetFileWriterOptions controls representable dataset file-writer settings.
type DatasetVersion ¶
DatasetVersion is one committed dataset version.
Metadata is a copy and remains valid after the Dataset is closed.
type Error ¶
Error is returned by dataset, writer, scanner and session operations.
type ExternalBlobMode ¶
type ExternalBlobMode int32
ExternalBlobMode selects how external blob references are written.
const ( // ExternalBlobModeReference stores external references without uploading. ExternalBlobModeReference ExternalBlobMode = 0 // ExternalBlobModeIngest uploads referenced payloads into managed storage. ExternalBlobModeIngest ExternalBlobMode = 1 )
type FTSBoostOptions ¶
type FTSBoostOptions struct {
NegativeBoost *float32
}
FTSBoostOptions configures a Boost query. A nil NegativeBoost keeps the native default.
type FTSCombinedFieldsOptions ¶
type FTSCombinedFieldsOptions struct {
Boosts []float32
Operator MatchOperator
}
FTSCombinedFieldsOptions configures a CombinedFields query. A nil or empty Boosts gives every field weight 1.
type FTSCoverage ¶
type FTSCoverage int32
FTSCoverage selects which fragments a prepared query may serve.
const ( // FTSCoverageStrict only serves fragments covered by the index. FTSCoverageStrict FTSCoverage = 0 // FTSCoverageIndexOnly rejects scans that would need unindexed fragments. FTSCoverageIndexOnly FTSCoverage = 1 )
type FTSDocumentGranularity ¶
type FTSDocumentGranularity int32
FTSDocumentGranularity selects how full-text matches map to rows.
const ( FTSDocumentUnspecified FTSDocumentGranularity = 0 FTSDocumentRow FTSDocumentGranularity = 1 FTSDocumentListElement FTSDocumentGranularity = 2 )
type FTSMatchOptions ¶
type FTSMatchOptions struct {
Boost *float32
Fuzziness int64
MaxExpansions *uint64
Operator MatchOperator
PrefixLength uint32
Granularity FTSDocumentGranularity
}
FTSMatchOptions configures a Match query. A nil Boost or MaxExpansions keeps the native default. Fuzziness 0 means exact matching and -1 selects native automatic fuzziness.
type FTSMatchPrepareOptions ¶
type FTSMatchPrepareOptions struct {
Operator MatchOperator
MaxFuzzyDistance uint32
Coverage FTSCoverage
}
FTSMatchPrepareOptions configures a prepared Match query.
type FTSMultiMatchOptions ¶
type FTSMultiMatchOptions struct {
Boosts []float32
Operator MatchOperator
}
FTSMultiMatchOptions configures a MultiMatch query. A nil or empty Boosts gives every column weight 1.
type FTSPhraseOptions ¶
type FTSPhraseOptions struct {
Slop uint32
Granularity FTSDocumentGranularity
}
FTSPhraseOptions configures a Phrase query.
type FTSPhrasePrepareOptions ¶
type FTSPhrasePrepareOptions struct {
Slop int32
Coverage FTSCoverage
}
FTSPhrasePrepareOptions configures a prepared Phrase query.
type FTSPreparedQuery ¶
type FTSPreparedQuery struct {
// contains filtered or unexported fields
}
FTSPreparedQuery is a query planned against one process-local dataset snapshot. It cannot be serialized, reused after reopening the dataset, or reused with another snapshot, even at the same URI and version.
func (*FTSPreparedQuery) Close ¶
func (q *FTSPreparedQuery) Close() error
Close releases the native prepared context. It is idempotent and waits for in-flight scanner attachment.
type FTSQuery ¶
type FTSQuery struct {
// contains filtered or unexported fields
}
FTSQuery is an immutable full-text query. Close releases the native query after any scanners that attached it have cloned it.
func NewFTSBooleanQuery ¶
NewFTSBooleanQuery combines queries with per-clause occurrence. Occurs and queries must have the same length; an empty query is valid at construction and rejected at planning.
func NewFTSBoostQuery ¶
func NewFTSBoostQuery(positive, negative *FTSQuery, options FTSBoostOptions) (*FTSQuery, error)
NewFTSBoostQuery builds a query that boosts or penalizes matches of positive and negative.
func NewFTSCombinedFieldsQuery ¶
func NewFTSCombinedFieldsQuery(terms string, columns []string, options FTSCombinedFieldsOptions) (*FTSQuery, error)
NewFTSCombinedFieldsQuery builds a CombinedFields query over explicit columns.
func NewFTSMatchQuery ¶
func NewFTSMatchQuery(terms, column string, options FTSMatchOptions) (*FTSQuery, error)
NewFTSMatchQuery builds a Match query. An empty column lets scan planning resolve the indexed column.
func NewFTSMultiMatchQuery ¶
func NewFTSMultiMatchQuery(terms string, columns []string, options FTSMultiMatchOptions) (*FTSQuery, error)
NewFTSMultiMatchQuery builds a MultiMatch query over explicit columns.
func NewFTSPhraseQuery ¶
func NewFTSPhraseQuery(terms, column string, options FTSPhraseOptions) (*FTSQuery, error)
NewFTSPhraseQuery builds a Phrase query. An empty column lets scan planning resolve the indexed column.
type FieldMetadataUpdate ¶
type FieldMetadataUpdate struct {
Field FieldReference
Updates []MetadataUpdate
Mode MetadataUpdateMode
}
FieldMetadataUpdate applies one metadata update set to a field.
type FieldReference ¶
type FieldReference struct {
Kind FieldReferenceKind
FieldID int32
Path string
}
FieldReference identifies a dataset field by stable field id or by nested path. FieldID is used for FieldReferenceByID; Path is used for FieldReferenceByPath.
type FieldReferenceKind ¶
type FieldReferenceKind int32
FieldReferenceKind selects how a FieldReference identifies a field.
const ( FieldReferenceByID FieldReferenceKind = 0 FieldReferenceByPath FieldReferenceKind = 1 )
type FieldStatistics ¶
FieldStatistics reports compressed on-disk bytes for one field.
type ForceStorageOptionsProvider ¶
type ForceStorageOptionsProvider = accessor.ForceProvider
ForceStorageOptionsProvider optionally handles an explicit force refresh. Without it, native force fetch falls back to StorageOptionsProvider.
type Fragment ¶
type Fragment struct {
// contains filtered or unexported fields
}
Fragment is a native Lance fragment snapshot. A fragment opened from a dataset and one returned by Fragment.Delete own a dataset snapshot and support scans, take, row counts and deletes. Fragments cloned from a transaction operation are detached: metadata is readable, but logical schema and data access returns CodeNotSupported.
func CreateFragment ¶
func CreateFragment(ctx context.Context, uri string, fragmentID uint64, reader array.RecordReader, opts ...FragmentCreateOption) (*Fragment, error)
CreateFragment writes one uncommitted fragment from reader and returns detached fragment metadata. The explicit fragmentID is used as-is; native code does not allocate it.
Creating a fragment does not publish a version. Publish the returned fragment by passing it to NewAppendOperation and committing the resulting transaction. If the transaction is never committed, the written data file remains in storage until it is cleaned up separately; closing the fragment never deletes it. An empty stream is rejected by native code.
Without WithFragmentSchema, native code infers the schema. With WithFragmentSchema, the native schema is cloned before the write and keeps field IDs that an Arrow schema cannot represent. The caller keeps ownership of reader: native code takes ownership of the exported stream and releases its retained reference on every return path. The caller must still release reader. A successful write exhausts the input; an error may leave it partially read, including batches prefetched during schema inference.
func WriteFragments ¶
func WriteFragments(ctx context.Context, uri string, reader array.RecordReader, opts ...FragmentCreateOption) ([]*Fragment, error)
WriteFragments writes zero or more uncommitted fragments and returns independently owned detached metadata. Native code splits the input using max_rows_per_file and other native write parameters. An empty input may produce no fragments.
Native code assigns final fragment IDs during commit. The IDs visible on the returned fragments before commit are temporary; do not rely on them being unique. Duplicate explicit IDs are rejected at commit, not here.
Publication, ownership and schema behavior are the same as CreateFragment.
func (*Fragment) AddColumnsBatchUDF ¶
func (f *Fragment) AddColumnsBatchUDF(ctx context.Context, udf BatchUDF) (*Fragment, *NativeSchema, error)
AddColumnsBatchUDF stages columns produced by udf. The mapper, output schema and checkpoint store use the same callbacks as Dataset.AddColumnsBatchUDF.
func (*Fragment) AddColumnsNulls ¶
func (f *Fragment) AddColumnsNulls(ctx context.Context, columns *arrow.Schema) (*Fragment, *NativeSchema, error)
AddColumnsNulls stages all-null columns described by columns. The schema supplies only the new logical field definitions; native allocates field IDs.
func (*Fragment) AddColumnsSQL ¶
func (f *Fragment) AddColumnsSQL(ctx context.Context, columns []SQLColumn, batchSize ...uint64) (*Fragment, *NativeSchema, error)
AddColumnsSQL stages SQL-computed columns on a clone of this fragment. The result fragment is detached metadata; publication remains an explicit operation and transaction step.
func (*Fragment) AddColumnsStream ¶
func (f *Fragment) AddColumnsStream(ctx context.Context, source array.RecordReader, batchSize ...uint64) (*Fragment, *NativeSchema, error)
AddColumnsStream stages columns read from source. Native consumes the transferred stream exactly once; the caller retains ownership of source.
func (*Fragment) CountRows ¶
CountRows counts rows matching filter. A nil filter counts all logical rows.
func (*Fragment) DataFileAt ¶
DataFileAt returns the data file at index.
func (*Fragment) DataFileCount ¶
DataFileCount returns the number of data files in the fragment.
func (*Fragment) DataFileForField ¶
DataFileForField returns the data file holding fieldID. The boolean is false when the fragment has no file for that field.
func (*Fragment) Delete ¶
Delete marks matching rows deleted and returns an independently owned, scannable snapshot with the staged deletion metadata. It returns (nil, nil) when every row in the fragment is deleted.
func (*Fragment) DeleteRows ¶
DeleteRows stages physical row offsets on a clone of this fragment. It returns nil when the offsets delete every physical row.
func (*Fragment) DeletedRows ¶
DeletedRows returns the number of rows hidden by deletion files.
func (*Fragment) LogicalRows ¶
LogicalRows returns PhysicalRows minus DeletedRows.
func (*Fragment) MergeColumns ¶
func (f *Fragment) MergeColumns(ctx context.Context, source array.RecordReader, leftOn, rightOn string) (*Fragment, *NativeSchema, error)
MergeColumns joins source into a clone of this fragment.
func (*Fragment) MetadataJSON ¶
func (f *Fragment) MetadataJSON() (json.RawMessage, error)
MetadataJSON returns the native fragment metadata as JSON.
func (*Fragment) NativeSchema ¶
func (f *Fragment) NativeSchema() (*NativeSchema, error)
NativeSchema clones the fragment's native schema, preserving field IDs and Lance-specific nested metadata that the Arrow view does not represent.
func (*Fragment) PhysicalRows ¶
PhysicalRows returns the fragment's physical row count.
func (*Fragment) Scan ¶
func (f *Fragment) Scan(opts ...ScanOption) (*Scanner, error)
Scan scans this fragment. The native fragment constructor does not accept WithColumns or WithSQLFilter; these return CodeNotSupported. The returned native scanner supports WithAdditionalSQLFilter, WithSubstraitFilter and WithSubstraitAggregate through its ordinary scanner setters. These options operate on this fragment snapshot and preserve native validation semantics.
func (*Fragment) Take ¶
func (f *Fragment) Take(ctx context.Context, indices []uint32, columns ...string) ([]arrow.RecordBatch, error)
Take takes fragment-local logical row indices. With no columns, all columns are returned. An explicit empty column list projects zero columns.
func (*Fragment) UpdateColumns ¶
func (f *Fragment) UpdateColumns(ctx context.Context, source array.RecordReader, leftOn, rightOn string) (*UpdateColumnsResult, error)
UpdateColumns stages a native update from source.
func (*Fragment) WriteColumnWithVersion ¶
func (f *Fragment) WriteColumnWithVersion(ctx context.Context, schema *NativeSchema, source array.RecordReader, version FragmentFileVersion) (*DataReplacementGroup, error)
WriteColumnWithVersion stages a replacement data file with an explicit file format. Native rejects FragmentFileVersionV1 and incompatible targets.
func (*Fragment) WriteColumns ¶
func (f *Fragment) WriteColumns(ctx context.Context, schema *NativeSchema, source array.RecordReader) (*DataReplacementGroup, error)
WriteColumns stages a replacement data file for schema.
type FragmentCreateOption ¶
type FragmentCreateOption = WriteOption
FragmentCreateOption configures CreateFragment and WriteFragments.
It aliases WriteOption so callers can reuse file-layout WriteParams, WithWriteStorageOptions, WithWriteSession and WithFragmentSchema. Creation does not commit: WithAutoCleanup, WithSkipAutoCleanup(true), stable row ID generation and transaction properties are rejected. Configure publication with CommitTransaction options, including WithCommitStableRowIDs and WithCommitSkipAutoCleanup. The native fragment API accepts static storage options only; WithWriteStorageOptionsAccessor and WithWriteBranch are rejected. Native-only WriteParams fields, including base placement, Blob policy, file-writer options and progress, are also rejected; use ordinary or staged dataset writes when those options are required.
type FragmentFileVersion ¶
type FragmentFileVersion int32
FragmentFileVersion selects the native file format for a staged replacement data file. The values match the native LanceFileFormatVersion discriminants.
const ( FragmentFileVersionV1 FragmentFileVersion = 0 FragmentFileVersionV2_0 FragmentFileVersion = 1 FragmentFileVersionV2_1 FragmentFileVersion = 2 FragmentFileVersionV2_2 FragmentFileVersion = 3 FragmentFileVersionV2_3 FragmentFileVersion = 4 )
type FragmentOffsets ¶
FragmentOffsets is one fragment entry in an UpdatedFragmentOffsets value. Offsets are forwarded as a set: repeated values collapse in native.
type IndexSegmentMetadata ¶
type IndexSegmentMetadata struct {
// contains filtered or unexported fields
}
IndexSegmentMetadata is an owned description of one distributed index segment. It borrows nothing from the bytes it was parsed from, so those bytes (typically a worker's uploaded artifact) can be discarded as soon as parsing succeeds. Close the metadata when it is no longer needed.
func ParseIndexSegmentMetadata ¶
func ParseIndexSegmentMetadata(data []byte) (*IndexSegmentMetadata, error)
ParseIndexSegmentMetadata decodes a segment metadata artifact produced by SegmentBuilder.Execute. The input is borrowed only for the duration of the call.
func (*IndexSegmentMetadata) BaseID ¶
func (m *IndexSegmentMetadata) BaseID() (uint32, bool, error)
BaseID reports the index base the segment extends. The boolean is false when no base is recorded.
func (*IndexSegmentMetadata) Clone ¶
func (m *IndexSegmentMetadata) Clone() (*IndexSegmentMetadata, error)
Clone returns an independent copy of the metadata.
func (*IndexSegmentMetadata) CloneWithFragmentIDs ¶
func (m *IndexSegmentMetadata) CloneWithFragmentIDs(fragmentIDs []uint32) (*IndexSegmentMetadata, error)
CloneWithFragmentIDs returns an independent copy whose coverage is replaced by fragmentIDs. A nil slice keeps the original coverage; a non-nil slice, including an empty one, sets it as given. The native side copies the in-memory value without re-validating coverage.
func (*IndexSegmentMetadata) Close ¶
func (m *IndexSegmentMetadata) Close() error
Close releases the metadata. It is safe to call more than once.
func (*IndexSegmentMetadata) CoveringFieldIDs ¶
func (m *IndexSegmentMetadata) CoveringFieldIDs() ([]int32, error)
CoveringFieldIDs reports the fields this segment's files can answer.
func (*IndexSegmentMetadata) CreatedAt ¶
func (m *IndexSegmentMetadata) CreatedAt() (time.Time, bool, error)
CreatedAt reports the segment creation time. The boolean is false when the segment carries no timestamp.
func (*IndexSegmentMetadata) DatasetVersion ¶
func (m *IndexSegmentMetadata) DatasetVersion() (uint64, error)
DatasetVersion reports the dataset version the segment was built from.
func (*IndexSegmentMetadata) Details ¶
func (m *IndexSegmentMetadata) Details() (json.RawMessage, bool, error)
Details returns the index-specific details payload. A present payload can be empty, which is distinct from an absent payload.
func (*IndexSegmentMetadata) DetailsTypeURL ¶
func (m *IndexSegmentMetadata) DetailsTypeURL() (string, bool, error)
DetailsTypeURL reports the encoded type URL of the index-specific details, if the segment carries details.
func (*IndexSegmentMetadata) Encode ¶
func (m *IndexSegmentMetadata) Encode() ([]byte, error)
Encode serializes the metadata back to its transport form. Encoding normalizes protobuf defaults, so the result can differ byte-wise from the input while describing the same value.
func (*IndexSegmentMetadata) FieldIDs ¶
func (m *IndexSegmentMetadata) FieldIDs() ([]int32, error)
FieldIDs reports the fields the index is keyed on.
func (*IndexSegmentMetadata) Files ¶
func (m *IndexSegmentMetadata) Files() ([]index.MetadataFile, error)
Files reports the physical files the segment references.
func (*IndexSegmentMetadata) FragmentIDs ¶
func (m *IndexSegmentMetadata) FragmentIDs() ([]uint32, error)
FragmentIDs reports the fragments the segment covers, in ascending order. An empty result means the segment covers no fragments explicitly.
func (*IndexSegmentMetadata) IndexVersion ¶
func (m *IndexSegmentMetadata) IndexVersion() (int, error)
IndexVersion reports the index format version. It is not a dataset version.
func (*IndexSegmentMetadata) Name ¶
func (m *IndexSegmentMetadata) Name() (string, error)
Name reports the index name the segment was built for.
func (*IndexSegmentMetadata) Type ¶
func (m *IndexSegmentMetadata) Type() (index.SegmentIndexType, error)
Type reports the segment's index family and concrete type. Scalar and vector type values come from disjoint enums, so one value identifies both.
type IndexSegmentResult ¶
type IndexSegmentResult struct {
// contains filtered or unexported fields
}
IndexSegmentResult is an immutable snapshot of one optimized index segment and the segments it replaces.
func IndexSegmentResultFromJSON ¶
func IndexSegmentResultFromJSON(data string) (*IndexSegmentResult, error)
IndexSegmentResultFromJSON restores an index segment result snapshot from native serialized JSON.
func (*IndexSegmentResult) Group ¶
func (r *IndexSegmentResult) Group() uint32
Group returns the merge group this result belongs to.
func (*IndexSegmentResult) JSON ¶
func (r *IndexSegmentResult) JSON() string
JSON returns the native serialized result JSON.
func (*IndexSegmentResult) PlanID ¶
func (r *IndexSegmentResult) PlanID() index.UUID
PlanID returns the plan identifier this result belongs to.
func (*IndexSegmentResult) ReadVersion ¶
func (r *IndexSegmentResult) ReadVersion() uint64
ReadVersion returns the dataset version the result read.
func (*IndexSegmentResult) ReplaceCount ¶
func (r *IndexSegmentResult) ReplaceCount() uint64
ReplaceCount returns the number of index segments the result replaces.
func (*IndexSegmentResult) ReplacedUUID ¶
func (r *IndexSegmentResult) ReplacedUUID(position uint64) (index.UUID, error)
ReplacedUUID returns one replaced segment's UUID.
func (*IndexSegmentResult) Segment ¶
func (r *IndexSegmentResult) Segment() (*IndexSegmentMetadata, error)
Segment returns an owned copy of the produced segment metadata. Close the metadata when it is no longer needed.
type MatchOperator ¶
type MatchOperator int32
MatchOperator combines the terms of a full-text query.
const ( // MatchOperatorOr matches documents containing any term. MatchOperatorOr MatchOperator = 0 // MatchOperatorAnd matches documents containing every term. MatchOperatorAnd MatchOperator = 1 )
type MaterializationStyle ¶
type MaterializationStyle int32
MaterializationStyle controls when projected columns are loaded.
const ( // MaterializationHeuristic lets the native planner choose per column. MaterializationHeuristic MaterializationStyle = 0 // MaterializationAllLate loads projected columns as late as possible. MaterializationAllLate MaterializationStyle = 1 // MaterializationAllEarly loads projected columns as early as possible. MaterializationAllEarly MaterializationStyle = 2 // MaterializationAllEarlyExcept loads all projected columns early except // the columns named in WithMaterializationStyle. MaterializationAllEarlyExcept MaterializationStyle = 3 )
type MergeInsertParams ¶
type MergeInsertParams struct {
WhenMatched MergeWhenMatched
WhenMatchedExpression string
WhenNotMatched MergeWhenNotMatched
WhenNotMatchedBySource MergeWhenNotMatchedBySource
WhenNotMatchedBySourceExpression string
}
MergeInsertParams controls MergeInsert. Its zero value is valid and selects find-or-create behavior: matching rows are kept and unmatched source rows are inserted.
The two expression fields are SQL filters. WhenMatchedExpression is required for MergeWhenMatchedUpdateIf and is invalid for every other when-matched mode. WhenNotMatchedBySourceExpression is required for MergeWhenNotMatchedBySourceDeleteIf and is invalid for every other not-matched-by-source mode.
type MergeInsertResult ¶
MergeInsertResult reports the rows affected by MergeInsert.
type MergeWhenMatched ¶
type MergeWhenMatched int32
MergeWhenMatched selects the action for a target row that matches a source row during MergeInsert.
const ( // MergeWhenMatchedDoNothing keeps the target row unchanged. It is the // zero value and produces find-or-create behavior. MergeWhenMatchedDoNothing MergeWhenMatched = 0 // MergeWhenMatchedUpdateAll replaces the target row with the source row. MergeWhenMatchedUpdateAll MergeWhenMatched = 1 // MergeWhenMatchedUpdateIf replaces the target row only when the // WhenMatchedExpression filter evaluates true. MergeWhenMatchedUpdateIf MergeWhenMatched = 2 // MergeWhenMatchedFail fails the operation when a target row matches. MergeWhenMatchedFail MergeWhenMatched = 3 // MergeWhenMatchedDelete removes the matching target row. MergeWhenMatchedDelete MergeWhenMatched = 4 )
type MergeWhenNotMatched ¶
type MergeWhenNotMatched int32
MergeWhenNotMatched selects the action for a source row with no matching target row during MergeInsert.
const ( // MergeWhenNotMatchedInsertAll inserts the source row. It is the zero // value. MergeWhenNotMatchedInsertAll MergeWhenNotMatched = 0 // MergeWhenNotMatchedDoNothing discards the source row. MergeWhenNotMatchedDoNothing MergeWhenNotMatched = 1 )
type MergeWhenNotMatchedBySource ¶
type MergeWhenNotMatchedBySource int32
MergeWhenNotMatchedBySource selects the action for a target row with no matching source row during MergeInsert.
const ( // MergeWhenNotMatchedBySourceKeep keeps the target row. It is the zero // value. MergeWhenNotMatchedBySourceKeep MergeWhenNotMatchedBySource = 0 // MergeWhenNotMatchedBySourceDelete removes every unmatched target row. MergeWhenNotMatchedBySourceDelete MergeWhenNotMatchedBySource = 1 // MergeWhenNotMatchedBySourceDeleteIf removes unmatched target rows that // satisfy WhenNotMatchedBySourceExpression. MergeWhenNotMatchedBySourceDeleteIf MergeWhenNotMatchedBySource = 2 )
type MetadataUpdate ¶
MetadataUpdate sets or removes one metadata key. A nil Value removes the key. An empty non-nil Value is a real empty string.
type MetadataUpdateMode ¶
type MetadataUpdateMode int32
MetadataUpdateMode controls whether metadata updates merge with or replace the existing metadata.
const ( // MetadataMerge updates only the keys present in the update. MetadataMerge MetadataUpdateMode = 0 // MetadataReplace replaces the complete metadata map with the update. MetadataReplace MetadataUpdateMode = 1 )
type Model ¶
Model is a trained index model array paired with the Arrow field it was trained with. Native index builders validate model provenance from that field's metadata, so reuse trained centroids or codebooks as a Model instead of the bare array. Model implements arrow.Array; Retain and Release operate on the embedded array.
func (*Model) ModelField ¶
ModelField returns the Arrow field, including provenance metadata, that the model was trained with.
type NativeSchema ¶
type NativeSchema struct {
// contains filtered or unexported fields
}
NativeSchema owns a native Lance schema, including field IDs that the lossy Arrow schema conversion does not represent. A NativeSchema stays usable after the dataset, fragment or operation that produced it is closed.
func (*NativeSchema) Arrow ¶
func (s *NativeSchema) Arrow() (*arrow.Schema, error)
Arrow exports a lossy Arrow view of the native schema.
func (*NativeSchema) Clone ¶
func (s *NativeSchema) Clone() (*NativeSchema, error)
Clone returns an independent native schema snapshot.
func (*NativeSchema) Close ¶
func (s *NativeSchema) Close() error
Close releases the schema. It is idempotent.
func (*NativeSchema) FieldID ¶
func (s *NativeSchema) FieldID(column string) (int32, error)
FieldID returns the native field ID for a column path.
func (*NativeSchema) Project ¶
func (s *NativeSchema) Project(columns ...string) (*NativeSchema, error)
Project returns a native schema containing the named top-level columns.
type OpenOption ¶
type OpenOption func(*openConfig) error
OpenOption configures Open.
func WithOpenCommitLock ¶
func WithOpenCommitLock(lock *CommitLock) OpenOption
WithOpenCommitLock routes Open through an external commit lock.
func WithSession ¶
func WithSession(session *Session) OpenOption
WithSession shares a session's caches with the dataset.
func WithStorageOptions ¶
func WithStorageOptions(options map[string]string) OpenOption
WithStorageOptions passes object-store options to Lance.
func WithStorageOptionsAccessor ¶
func WithStorageOptionsAccessor(accessor *StorageOptionsAccessor) OpenOption
WithStorageOptionsAccessor supplies a reusable dynamic storage-options accessor. It is mutually exclusive with WithStorageOptions and is safe to close after Open returns; native datasets retain their own reference.
func WithVersion ¶
func WithVersion(version uint64) OpenOption
WithVersion selects a dataset version. Zero selects the latest version.
type Operation ¶
type Operation struct {
// contains filtered or unexported fields
}
Operation is a native transaction operation snapshot.
func NewAppendOperation ¶
NewAppendOperation clones fragments into an Append operation.
func NewDataReplacementOperation ¶
func NewDataReplacementOperation(groups ...*DataReplacementGroup) (*Operation, error)
NewDataReplacementOperation constructs a DataReplacement operation.
func NewDeleteOperation ¶
func NewDeleteOperation(updated []*Fragment, deletedFragmentIDs []uint64, predicate string) (*Operation, error)
NewDeleteOperation clones updated fragments and deleted IDs into a Delete operation.
func NewMergeOperation ¶
func NewMergeOperation(schema *NativeSchema, fragments []*Fragment, preservesNullability bool) (*Operation, error)
NewMergeOperation clones a complete final fragment list and its complete schema into a Merge operation. The caller keeps ownership of schema and fragments.
func NewProjectOperation ¶
func NewProjectOperation(schema *NativeSchema, preservesNullability bool) (*Operation, error)
NewProjectOperation constructs a Project operation.
func NewReserveFragmentsOperation ¶
NewReserveFragmentsOperation constructs a ReserveFragments operation.
func NewRestoreOperation ¶
NewRestoreOperation constructs a Restore operation.
func NewUpdateBasesOperation ¶
NewUpdateBasesOperation constructs an UpdateBases operation.
func NewUpdateOperation ¶
func NewUpdateOperation(options UpdateOperationOptions) (*Operation, error)
NewUpdateOperation clones updated fragments and metadata into an Update operation. It always selects native RewriteColumns mode because Fragment.UpdateColumns produces a horizontal column rewrite. The caller keeps ownership of every input.
func (*Operation) AppendFragment ¶
AppendFragment returns one detached fragment from an Append operation.
func (*Operation) AppendFragmentCount ¶
AppendFragmentCount returns the number of fragments in an Append operation.
func (*Operation) AppendFragments ¶
AppendFragments returns every detached fragment from an Append operation.
func (*Operation) DataReplacement ¶
func (o *Operation) DataReplacement(index uint64) (*DataReplacementGroup, error)
DataReplacement returns one cloned data replacement group.
func (*Operation) DataReplacementCount ¶
DataReplacementCount returns the number of data replacement groups.
func (*Operation) DataReplacements ¶
func (o *Operation) DataReplacements() ([]*DataReplacementGroup, error)
DataReplacements returns every data replacement group.
func (*Operation) DeleteDeletedFragmentCount ¶
DeleteDeletedFragmentCount returns the number of fully deleted fragment IDs.
func (*Operation) DeleteDeletedFragmentID ¶
DeleteDeletedFragmentID returns one fully deleted fragment ID.
func (*Operation) DeleteDeletedFragmentIDs ¶
DeleteDeletedFragmentIDs returns every fully deleted fragment ID.
func (*Operation) DeletePredicate ¶
DeletePredicate returns the Delete operation's SQL predicate.
func (*Operation) DeleteUpdatedFragmentCount ¶
DeleteUpdatedFragmentCount returns the number of updated Delete fragments.
func (*Operation) DeleteUpdatedFragmentMetadata ¶
func (o *Operation) DeleteUpdatedFragmentMetadata(index uint64) (json.RawMessage, error)
DeleteUpdatedFragmentMetadata returns one updated fragment's metadata JSON.
func (*Operation) FragmentCount ¶
FragmentCount returns the fragment count for Append or Overwrite.
func (*Operation) Kind ¶
func (o *Operation) Kind() (OperationKind, error)
Kind returns the operation variant.
func (*Operation) MergeFragment ¶
MergeFragment returns one detached fragment from a Merge operation.
func (*Operation) MergeFragmentCount ¶
MergeFragmentCount returns the number of fragments in a Merge operation.
func (*Operation) MergeFragments ¶
MergeFragments returns every detached fragment from a Merge operation.
func (*Operation) NativeSchema ¶
func (o *Operation) NativeSchema() (*NativeSchema, error)
NativeSchema clones the in-memory native operation schema.
func (*Operation) PreservesNullability ¶
PreservesNullability reads the assertion from a Merge or Project operation.
func (*Operation) ReserveFragmentsCount ¶
ReserveFragmentsCount returns the reserved fragment count.
func (*Operation) RestoreVersion ¶
RestoreVersion returns the target version of a Restore operation.
func (*Operation) Schema ¶
Schema exports a lossy Arrow view of an Overwrite, Merge or Project schema.
func (*Operation) UpdateBase ¶
UpdateBase returns one base path from an UpdateBases operation.
func (*Operation) UpdateBases ¶
UpdateBases returns every base path from an UpdateBases operation.
func (*Operation) UpdateBasesCount ¶
UpdateBasesCount returns the number of new base paths.
func (*Operation) UpdateFieldsModified ¶
UpdateFieldsModified returns the native modified field IDs.
func (*Operation) UpdateUpdatedFragment ¶
UpdateUpdatedFragment returns one detached updated fragment.
func (*Operation) UpdateUpdatedFragmentCount ¶
UpdateUpdatedFragmentCount returns the number of updated fragments.
func (*Operation) UpdateUpdatedFragmentOffsets ¶
func (o *Operation) UpdateUpdatedFragmentOffsets() (*UpdatedFragmentOffsets, bool, error)
UpdateUpdatedFragmentOffsets returns the native offsets and whether they were present. A present value with zero entries is distinct from absent.
func (*Operation) UpdateUpdatedFragments ¶
UpdateUpdatedFragments returns every detached updated fragment.
type OperationKind ¶
type OperationKind int32
OperationKind identifies a native transaction operation variant.
const ( OperationUnknown OperationKind = 0 OperationAppend OperationKind = 1 OperationDelete OperationKind = 2 OperationOverwrite OperationKind = 3 OperationCreateIndex OperationKind = 4 OperationRewrite OperationKind = 5 OperationDataReplacement OperationKind = 6 OperationDataOverlay OperationKind = 7 OperationMerge OperationKind = 8 OperationRestore OperationKind = 9 OperationReserveFragments OperationKind = 10 OperationUpdate OperationKind = 11 OperationProject OperationKind = 12 OperationUpdateConfig OperationKind = 13 OperationUpdateMemWALState OperationKind = 14 OperationClone OperationKind = 15 OperationUpdateBases OperationKind = 16 )
type OptimizeIndexMerge ¶
type OptimizeIndexMerge struct {
// contains filtered or unexported fields
}
OptimizeIndexMerge is an immutable snapshot of one merge group. Native merge execution validates the dataset version and plan ID, and selects supplied task results belonging to this merge group. Results from other groups in the same plan are ignored.
func OptimizeIndexMergeFromJSON ¶
func OptimizeIndexMergeFromJSON(data string) (*OptimizeIndexMerge, error)
OptimizeIndexMergeFromJSON restores a merge snapshot from native serialized JSON.
func (*OptimizeIndexMerge) Execute ¶
func (m *OptimizeIndexMerge) Execute(ctx context.Context, dataset *Dataset, results []*IndexSegmentResult) (*IndexSegmentResult, error)
Execute runs the merge against a dataset snapshot and task results. It writes index files but publishes no dataset version. A nil result with a nil error means native optimization found no work for this group.
Native validation remains authoritative: results from another plan and a dataset that cannot be read at the merge's read version are rejected. Results from other groups in the same plan are ignored.
func (*OptimizeIndexMerge) Group ¶
func (m *OptimizeIndexMerge) Group() uint32
Group returns the merge group.
func (*OptimizeIndexMerge) JSON ¶
func (m *OptimizeIndexMerge) JSON() string
JSON returns the native serialized merge JSON.
func (*OptimizeIndexMerge) OldSegment ¶
func (m *OptimizeIndexMerge) OldSegment(index uint64) (*IndexSegmentMetadata, error)
OldSegment returns an owned copy of one old segment's metadata. Close the metadata when it is no longer needed.
func (*OptimizeIndexMerge) OldSegmentCount ¶
func (m *OptimizeIndexMerge) OldSegmentCount() uint64
OldSegmentCount returns the number of old index segments the merge folds into its output.
func (*OptimizeIndexMerge) PlanID ¶
func (m *OptimizeIndexMerge) PlanID() index.UUID
PlanID returns the plan identifier this merge belongs to.
func (*OptimizeIndexMerge) ReadVersion ¶
func (m *OptimizeIndexMerge) ReadVersion() uint64
ReadVersion returns the dataset version the merge reads.
func (*OptimizeIndexMerge) ReplaceCount ¶
func (m *OptimizeIndexMerge) ReplaceCount() uint64
ReplaceCount returns the number of index segments the merge replaces.
func (*OptimizeIndexMerge) ReplacedUUID ¶
func (m *OptimizeIndexMerge) ReplacedUUID(position uint64) (index.UUID, error)
ReplacedUUID returns one replaced segment's UUID.
type OptimizeIndexTask ¶
type OptimizeIndexTask struct {
// contains filtered or unexported fields
}
OptimizeIndexTask is an immutable snapshot of one build task. It carries the task JSON, source fragment coverage, reference and old segment metadata copies, and the task's native merge options.
func OptimizeIndexTaskFromJSON ¶
func OptimizeIndexTaskFromJSON(data string) (*OptimizeIndexTask, error)
OptimizeIndexTaskFromJSON restores a build task snapshot from native serialized JSON.
func (*OptimizeIndexTask) EnablePartitionRebalancing ¶
func (t *OptimizeIndexTask) EnablePartitionRebalancing() bool
EnablePartitionRebalancing reports whether the task may rebalance partitions.
func (*OptimizeIndexTask) Execute ¶
func (t *OptimizeIndexTask) Execute(ctx context.Context, dataset *Dataset) (*IndexSegmentResult, error)
Execute runs the task against a dataset snapshot. It writes index files but publishes no dataset version. A nil result with a nil error means native optimization found no work for this task.
ctx is checked before work starts; it is not observed once native execution is in progress.
func (*OptimizeIndexTask) FragmentCount ¶
func (t *OptimizeIndexTask) FragmentCount() uint64
FragmentCount returns the number of source fragments the task covers.
func (*OptimizeIndexTask) FragmentID ¶
func (t *OptimizeIndexTask) FragmentID(index uint64) (uint32, error)
FragmentID returns one source fragment id.
func (*OptimizeIndexTask) Group ¶
func (t *OptimizeIndexTask) Group() uint32
Group returns the merge group this task feeds.
func (*OptimizeIndexTask) JSON ¶
func (t *OptimizeIndexTask) JSON() string
JSON returns the native serialized task JSON.
func (*OptimizeIndexTask) NumIndicesToMerge ¶
func (t *OptimizeIndexTask) NumIndicesToMerge() compaction.Optional[uint64]
NumIndicesToMerge returns the task's optional merge limit.
func (*OptimizeIndexTask) OldSegment ¶
func (t *OptimizeIndexTask) OldSegment(index uint64) (*IndexSegmentMetadata, error)
OldSegment returns an owned copy of one old segment's metadata. Close the metadata when it is no longer needed.
func (*OptimizeIndexTask) OldSegmentCount ¶
func (t *OptimizeIndexTask) OldSegmentCount() uint64
OldSegmentCount returns the number of old index segments the task may fold into its output.
func (*OptimizeIndexTask) PlanID ¶
func (t *OptimizeIndexTask) PlanID() index.UUID
PlanID returns the plan identifier this task belongs to.
func (*OptimizeIndexTask) ReadVersion ¶
func (t *OptimizeIndexTask) ReadVersion() uint64
ReadVersion returns the dataset version the task reads.
func (*OptimizeIndexTask) Reference ¶
func (t *OptimizeIndexTask) Reference() (*IndexSegmentMetadata, error)
Reference returns an owned copy of the task's reference segment metadata. Close the metadata when it is no longer needed.
func (*OptimizeIndexTask) Retrain ¶
func (t *OptimizeIndexTask) Retrain() bool
Retrain reports whether the task retrains a vector index.
type OptimizeIndicesPlan ¶
type OptimizeIndicesPlan struct {
// contains filtered or unexported fields
}
OptimizeIndicesPlan is an immutable snapshot of a native index-optimization plan. It carries the plan ID, read version, task and merge snapshots, and the native serialized plan JSON. A plan does not reserve files or publish anything.
The native plan, task and merge JSON is authoritative across a transport boundary. Native JSON does not serialize OptimizeOptions, so transaction properties must be passed again to CommitOptimizeIndices.
func OptimizeIndicesPlanFromJSON ¶
func OptimizeIndicesPlanFromJSON(data string) (*OptimizeIndicesPlan, error)
OptimizeIndicesPlanFromJSON restores an index-optimization plan snapshot from native serialized JSON. All task, merge and metadata snapshots are copied eagerly. OptimizeOptions are not part of the native JSON.
func (*OptimizeIndicesPlan) JSON ¶
func (p *OptimizeIndicesPlan) JSON() string
JSON returns the native serialized plan JSON.
func (*OptimizeIndicesPlan) Merge ¶
func (p *OptimizeIndicesPlan) Merge(index uint64) (*OptimizeIndexMerge, error)
Merge returns one merge group snapshot.
func (*OptimizeIndicesPlan) MergeCount ¶
func (p *OptimizeIndicesPlan) MergeCount() uint64
MergeCount returns the number of merge groups in the plan.
func (*OptimizeIndicesPlan) PlanID ¶
func (p *OptimizeIndicesPlan) PlanID() index.UUID
PlanID returns the native plan identifier.
func (*OptimizeIndicesPlan) ReadVersion ¶
func (p *OptimizeIndicesPlan) ReadVersion() uint64
ReadVersion returns the dataset version the plan was created from.
func (*OptimizeIndicesPlan) Task ¶
func (p *OptimizeIndicesPlan) Task(index uint64) (*OptimizeIndexTask, error)
Task returns one build task snapshot.
func (*OptimizeIndicesPlan) TaskCount ¶
func (p *OptimizeIndicesPlan) TaskCount() uint64
TaskCount returns the number of build tasks in the plan.
type OrderDirection ¶
type OrderDirection int32
OrderDirection is an ascending or descending sort direction.
const ( Ascending OrderDirection = 0 Descending OrderDirection = 1 )
type Ordering ¶
type Ordering struct {
Column string
Direction OrderDirection
NullOrder NullOrder
}
Ordering is one scanner sort key.
type ProgressEvent ¶
type ProgressEvent = index.ProgressEvent
ProgressEvent identifies the kind of a progress observation.
type ProjectionColumn ¶
ProjectionColumn selects an output column by SQL expression.
type Ref ¶
type Ref struct {
// contains filtered or unexported fields
}
Ref identifies a dataset state for CheckoutRef and branch or tag creation.
Construct references with VersionRef, BranchRef, BranchVersionRef, or TagRef. The zero Ref is invalid and is rejected by native calls.
func BranchRef ¶
BranchRef selects the latest version of a branch. The special name "main" is normalized by native code.
func BranchVersionRef ¶
BranchVersionRef selects a specific version on a branch. version is relative to the branch and is not an absolute dataset version.
func VersionRef ¶
VersionRef selects a version number resolved on the branch the receiver is checked out on. On a main-branch dataset this is the absolute dataset version; on another branch it is that branch's version number.
type RewriteResult ¶
type RewriteResult struct {
// contains filtered or unexported fields
}
RewriteResult is an immutable snapshot of a compaction task rewrite.
func RewriteResultFromJSON ¶
func RewriteResultFromJSON(data string) (*RewriteResult, error)
RewriteResultFromJSON restores a rewrite result from native serialized JSON.
func (*RewriteResult) JSON ¶
func (r *RewriteResult) JSON() string
JSON returns the native serialized rewrite result JSON.
func (*RewriteResult) Metrics ¶
func (r *RewriteResult) Metrics() compaction.Metrics
Metrics returns the rewrite metrics reported by native execution.
func (*RewriteResult) NewFragmentCount ¶
func (r *RewriteResult) NewFragmentCount() uint64
NewFragmentCount returns the number of rewritten fragments.
func (*RewriteResult) NewFragmentMetadataJSON ¶
func (r *RewriteResult) NewFragmentMetadataJSON(index uint64) (string, error)
NewFragmentMetadataJSON returns one rewritten fragment's metadata JSON.
func (*RewriteResult) OriginalFragmentCount ¶
func (r *RewriteResult) OriginalFragmentCount() uint64
OriginalFragmentCount returns the number of source fragments.
func (*RewriteResult) OriginalFragmentMetadataJSON ¶
func (r *RewriteResult) OriginalFragmentMetadataJSON(index uint64) (string, error)
OriginalFragmentMetadataJSON returns one source fragment's metadata JSON.
func (*RewriteResult) ReadVersion ¶
func (r *RewriteResult) ReadVersion() uint64
ReadVersion returns the dataset version the result was produced from.
type ScanMetric ¶
ScanMetric is one native scanner metric.
type ScanOption ¶
type ScanOption func(*scanConfig) error
ScanOption configures a scanner.
func WithAdditionalSQLFilter ¶
func WithAdditionalSQLFilter(filter string) ScanOption
WithAdditionalSQLFilter combines another SQL predicate with the filter.
func WithBatchReadahead ¶
func WithBatchReadahead(batches uint64) ScanOption
WithBatchReadahead sets how many batches the native scanner reads ahead.
func WithBatchSize ¶
func WithBatchSize(size int64) ScanOption
WithBatchSize sets the target row count per batch.
func WithBatchSizeBytes ¶
func WithBatchSizeBytes(size uint64) ScanOption
WithBatchSizeBytes sets the target byte size per batch.
func WithBlobHandling ¶
func WithBlobHandling(handling BlobHandling) ScanOption
WithBlobHandling selects how blob columns are represented.
func WithColumns ¶
func WithColumns(columns ...string) ScanOption
WithColumns selects columns by name.
func WithFTSIndexSegments ¶
func WithFTSIndexSegments(uuids ...index.UUID) ScanOption
WithFTSIndexSegments restricts an FTS scan to the given physical index segments. A nonempty selection requires WithFTSQueryContext. No arguments clears the selection. Native planning rejects unknown UUIDs.
func WithFTSQuery ¶
func WithFTSQuery(query *FTSQuery) ScanOption
WithFTSQuery attaches an immutable full-text query to the scan.
func WithFTSQueryContext ¶
func WithFTSQueryContext(query *FTSPreparedQuery) ScanOption
WithFTSQueryContext attaches a prepared full-text query context. The context must come from this exact process-local dataset snapshot.
func WithFragmentIDs ¶
func WithFragmentIDs(fragmentIDs ...uint64) ScanOption
WithFragmentIDs restricts the scan to the given fragment IDs. Calling it with no IDs selects no fragments. It replaces the metadata selection staged by Fragment.Scan, including staged deletion metadata.
func WithFragmentReadahead ¶
func WithFragmentReadahead(fragments uint64) ScanOption
WithFragmentReadahead sets how many fragments the native scanner reads ahead. It has no effect when WithScanInOrder(true).
func WithIOBufferSize ¶
func WithIOBufferSize(size uint64) ScanOption
WithIOBufferSize sets the native scanner I/O buffer size in bytes.
func WithIncludeDeletedRows ¶
func WithIncludeDeletedRows(enabled bool) ScanOption
WithIncludeDeletedRows includes rows deleted from the scanned snapshot. WithRowID(true) is required. Deleted rows have a null _rowid. For filtered scans, use WithUseScalarIndex(false) to prevent index plans that omit tombstoned rows. Explicit scalar index segments are incompatible.
func WithLimit ¶
func WithLimit(limit int64) ScanOption
WithLimit limits the number of returned rows.
func WithMaterializationStyle ¶
func WithMaterializationStyle(style MaterializationStyle, earlyExcept ...string) ScanOption
WithMaterializationStyle selects when projected columns are loaded. earlyExcept names Lance schema paths to load late when style is MaterializationAllEarlyExcept. An empty exception list with that style loads every projected column early.
func WithMinHashQuery ¶
func WithMinHashQuery(column, text string) ScanOption
WithMinHashQuery attaches a MinHash LSH similarity query on column. The scan still needs an explicit limit and any desired filters, matching native scanner behavior.
func WithNearest ¶
func WithNearest(column string, query Vector, k uint32) ScanOption
WithNearest returns the k nearest rows to a single query vector. The column must have the same vector dimension as query. Native also accepts Float32 queries for Float16 and Float64 columns by coercing their element type; other combinations require matching element types. k must be positive.
This option is mutually exclusive with WithNearestBatch and WithNearestMultiVector; when several are given, the last one wins. A scan with a nearest query is ordered by distance and returns the _distance column.
func WithNearestBatch ¶
func WithNearestBatch(column string, queries Vector, dimension int, k uint32) ScanOption
WithNearestBatch returns the k nearest rows for each row of a row-major query matrix. queries holds numQueries*dimension elements, where dimension is the query vector dimension. The column must contain FixedSizeList vectors of the same element type and dimension.
The result contains the query_index column, which identifies the query that produced each row. Results for different queries have no global ordering. At most 128 queries and 64 MiB of query data are accepted. numQueries*k, multiplied by refineFactor when set, must not exceed 100000. A global limit or offset cannot be combined with a batch query.
func WithNearestMultiVector ¶
func WithNearestMultiVector(column string, vectors Vector, dimension int, k uint32) ScanOption
WithNearestMultiVector returns the nearest rows for a multi-vector query. vectors holds numVectors*dimension elements, where dimension is the dimension of each subvector. The column must contain List values whose elements are FixedSizeList vectors of dimension with float16, float32, or float64 elements.
At most 128 subvectors are accepted, and numVectors*k and k*refineFactor must each not exceed 100000. The result contains the nearest rows for the combined query. WithLimit and WithOffset apply to these combined results through the native result window.
func WithOrderBy ¶
func WithOrderBy(orderings ...Ordering) ScanOption
WithOrderBy appends native sort keys.
func WithProjection ¶
func WithProjection(columns ...ProjectionColumn) ScanOption
WithProjection selects output columns by SQL expression.
func WithRowAddress ¶
func WithRowAddress(enabled bool) ScanOption
WithRowAddress includes Lance row addresses in scanned batches.
func WithRowID ¶
func WithRowID(enabled bool) ScanOption
WithRowID includes Lance row IDs in scanned batches.
func WithRowIDAllowlist ¶
func WithRowIDAllowlist(rowIDs ...uint64) ScanOption
WithRowIDAllowlist restricts the scan to the given stable row IDs. The IDs are in the dataset's _rowid space; they are not positions within a batch or fragment. Calling it with no IDs selects no rows.
func WithRowIDBlocklist ¶
func WithRowIDBlocklist(rowIDs ...uint64) ScanOption
WithRowIDBlocklist excludes the given stable row IDs from the scan. The IDs are in the dataset's _rowid space; they are not positions within a batch or fragment.
func WithSQLFilter ¶
func WithSQLFilter(filter string) ScanOption
WithSQLFilter sets the scanner SQL filter.
func WithScalarIndexSegment ¶
func WithScalarIndexSegment(segment index.UUID) ScanOption
WithScalarIndexSegment restricts a scalar-index search to one index segment. The scan must also set at least one explicit fragment ID; those fragments define both the segment read domain and the fallback domain for rows not covered by the segment.
func WithScanInOrder ¶
func WithScanInOrder(enabled bool) ScanOption
WithScanInOrder requires fragments to be returned in dataset order. It is enabled by default.
func WithScanStatistics ¶
func WithScanStatistics(callback func(ScanStatistics)) ScanOption
WithScanStatistics registers a callback for completed scans.
func WithStrictBatchSize ¶
func WithStrictBatchSize(enabled bool) ScanOption
WithStrictBatchSize requires every batch except the last to contain exactly the configured batch size. It cannot be combined with WithBatchSizeBytes.
func WithSubstraitAggregate ¶
func WithSubstraitAggregate(data []byte) ScanOption
WithSubstraitAggregate sets a serialized Substrait aggregate expression.
func WithSubstraitFilter ¶
func WithSubstraitFilter(data []byte) ScanOption
WithSubstraitFilter sets a serialized Substrait filter.
func WithTargetParallelism ¶
func WithTargetParallelism(tasks uint64) ScanOption
WithTargetParallelism sets the native scanner's target task parallelism.
func WithUseScalarIndex ¶
func WithUseScalarIndex(enabled bool) ScanOption
WithUseScalarIndex controls whether scalar indexes may optimize filters. It is enabled by default and is independent of WithVectorUseIndex.
func WithUseStats ¶
func WithUseStats(enabled bool) ScanOption
WithUseStats controls whether file statistics may optimize the scan. It is enabled by default.
func WithVectorApproxMode ¶
func WithVectorApproxMode(mode ApproxMode) ScanOption
WithVectorApproxMode selects the vector index accuracy mode.
func WithVectorDistanceRange ¶
func WithVectorDistanceRange(lower, upper *float32) ScanOption
WithVectorDistanceRange restricts a single-vector nearest search to lower <= distance < upper. A nil bound is unbounded. Bounds must be finite, and lower must be less than upper when both are set. L2 distance is squared Euclidean distance.
func WithVectorEF ¶
func WithVectorEF(ef uint32) ScanOption
WithVectorEF sets the HNSW ef search parameter.
func WithVectorFastSearch ¶
func WithVectorFastSearch(fast bool) ScanOption
WithVectorFastSearch enables index-only search with weak consistency. Rows that are not yet visible to the index may be missing, and the search can return no rows when no usable index exists. It does not fall back to an exact scan, even with WithVectorUseIndex(false).
func WithVectorMaxNProbes ¶
func WithVectorMaxNProbes(n uint32) ScanOption
WithVectorMaxNProbes sets the maximum number of IVF partitions to probe.
func WithVectorMetric ¶
func WithVectorMetric(metric VectorMetric) ScanOption
WithVectorMetric selects the distance metric for the scan, overriding the dataset or index metric. Hamming is only valid for UInt8 columns.
func WithVectorMinNProbes ¶
func WithVectorMinNProbes(n uint32) ScanOption
WithVectorMinNProbes sets the minimum number of IVF partitions to probe.
func WithVectorNProbes ¶
func WithVectorNProbes(n uint32) ScanOption
WithVectorNProbes sets the number of IVF partitions to probe. It cannot be combined with WithVectorMinNProbes or WithVectorMaxNProbes.
func WithVectorPrefilter ¶
func WithVectorPrefilter(prefilter bool) ScanOption
WithVectorPrefilter controls whether the SQL filter is applied before or after vector ranking. Prefiltering before ranking yields exact filtered nearest neighbors; post-filtering only filters the ranked candidates.
func WithVectorQueryParallelism ¶
func WithVectorQueryParallelism(n int32) ScanOption
WithVectorQueryParallelism sets the number of batches searched in parallel. A value of -1 lets the native implementation choose.
func WithVectorRefineFactor ¶
func WithVectorRefineFactor(factor uint32) ScanOption
WithVectorRefineFactor re-ranks k*factor candidates using the original vector values. The factor must be positive. It is only meaningful for indexed searches.
func WithVectorSearchEffort ¶
func WithVectorSearchEffort(effort float64) ScanOption
WithVectorSearchEffort trades accuracy for speed in [0,1], where a higher value searches more thoroughly. An explicit effort overrides a fixed nprobes count, while explicit minimum and maximum nprobes bounds remain effective.
func WithVectorUseIndex ¶
func WithVectorUseIndex(use bool) ScanOption
WithVectorUseIndex controls whether vector indexes may be used.
func WithoutScoringAutoprojection ¶
func WithoutScoringAutoprojection() ScanOption
WithoutScoringAutoprojection disables automatic _score and _distance output columns for FTS and vector scans. After this option, those columns are returned only when explicitly projected. The setting cannot be undone on the same scanner.
type ScanStatistics ¶
type ScanStatistics struct {
IOPS uint64
Requests uint64
BytesRead uint64
IndicesLoaded uint64
IndexPartitionsLoaded uint64
IndexComparisons uint64
Metrics []ScanMetric
}
ScanStatistics is a snapshot of native scan statistics.
type Scanner ¶
type Scanner struct {
// contains filtered or unexported fields
}
Scanner is a single-use native scan. Records and ToArrowReader may each be called at most once, and not both. Close is safe concurrently with scanning.
func (*Scanner) Records ¶
Records returns an iterator over scanned batches. Each yielded batch is borrowed and is released when iteration advances; call Retain to keep it. Breaking out of the loop closes the scanner.
func (*Scanner) Statistics ¶
func (s *Scanner) Statistics() ScanStatistics
Statistics returns a snapshot of native scan statistics.
func (*Scanner) ToArrowReader ¶
ToArrowReader returns an Arrow RecordReader over the scan. The reader owns the scanner and closes it when Release reaches zero.
type SegmentBuilder ¶
type SegmentBuilder struct {
// contains filtered or unexported fields
}
SegmentBuilder builds one physical index segment without committing it. The resulting metadata bytes can be moved to another process, inspected, merged, or committed together with other segments through CommitIndexSegments.
A builder owns a private snapshot of the dataset it was created from, so the originating Dataset may be closed before the builder executes. A builder executes at most once and must be closed when it is no longer needed.
func (*SegmentBuilder) Close ¶
func (b *SegmentBuilder) Close() error
Close releases the builder. It waits for an in-flight Execute or SetProgress to finish and is safe to call more than once.
func (*SegmentBuilder) Execute ¶
func (b *SegmentBuilder) Execute(ctx context.Context) ([]byte, error)
Execute builds the segment and returns its encoded index metadata. The bytes are a self-contained transport value: they can be written to disk, parsed with ParseIndexSegmentMetadata, or committed by any process that opens a dataset containing the segment files. Execute does not commit a dataset version, and no native work can be cancelled once it starts; a cancelled context is only observed before execution begins.
Execute may be called once, and the builder stays consumed even when execution fails. A failed execution leaves the builder unusable; close it and build a new one.
func (*SegmentBuilder) SetProgress ¶
func (b *SegmentBuilder) SetProgress(fn index.ProgressFunc) error
SetProgress installs a progress observer for the next execution, replacing a previously installed observer. The observer runs on native worker threads and may be called concurrently; it must not block, call back into lance, or panic. A nil observer is rejected: builders without an observer simply skip these calls. Progress is advisory and never affects the build result.
type Session ¶
type Session = sharesession.Session
Session owns shared index and metadata caches. Datasets and raw file objects opened with a session keep their cache state after the session is closed.
func NewSession ¶
NewSession creates a session with fixed-size caches.
func NewSessionWithCacheSpecs ¶
NewSessionWithCacheSpecs creates a session with configured caches.
type StorageOptionsAccessor ¶
StorageOptionsAccessor is a reusable owned accessor for dynamic storage options. It may be supplied to Open, Write, WriteUncommitted, CommitTransaction and CommitTransactionBatch, and to raw-file readers, writers and sessions.
The accessor contains its initial static options and optional provider. Closing it waits for in-flight local snapshot calls but does not wait for native datasets or stores that retained their own reference. Callers own the provider value passed to NewStorageOptionsAccessor; the accessor does not call a Go provider Close method.
func NewNamespaceStorageOptionsAccessor ¶
func NewNamespaceStorageOptionsAccessor(ns namespace.Namespace, tableID []string) (*StorageOptionsAccessor, error)
NewNamespaceStorageOptionsAccessor creates a table-scoped accessor backed by native namespace credential vending. It is intended for raw-file readers, writers and sessions that need the same credentials as namespace-managed tables.
The provider retains the native namespace. Closing the public namespace handle does not invalidate the accessor or datasets and stores that already retained it; the native namespace is released by its final native owner. tableID uses the same typed string IDs as OpenFromNamespace and friends.
func NewStorageOptionsAccessor ¶
func NewStorageOptionsAccessor(initial map[string]string, provider StorageOptionsProvider) (*StorageOptionsAccessor, error)
NewStorageOptionsAccessor creates a reusable accessor. A nil initial map means native None; a non-nil map, including an empty one, means Some(initial). Native uses valid initial options without fetching. After expiration, provider options replace the initial set; a nil result falls back to the initial set.
type StorageOptionsProvider ¶
StorageOptionsProvider supplies dynamic storage options to Lance.
StorageOptionsProviderID defines native provider identity, equality and object-store registry cache scope. Providers that expose different credential scopes must use different IDs; sharing an ID is an explicit decision to share the native store and its cached credentials.
FetchStorageOptions may be called concurrently from native worker threads. It returns nil for native absence and a non-nil empty map for an explicitly empty option set. Native code owns expiration, caching and fallback; a provider must not start a refresh goroutine or retain fetched credentials in a second Go cache.
type Tag ¶
type Tag struct {
Name string
Branch string
Version uint64
CreatedAt *time.Time
UpdatedAt *time.Time
ManifestSize uint64
Metadata map[string]string
}
Tag is a named pointer to a dataset version.
Tags are stored in a catalog shared by every dataset that uses the same storage location. A Tag is a copied snapshot and remains valid after the Dataset is closed.
type Transaction ¶
type Transaction struct {
// contains filtered or unexported fields
}
Transaction is an owned native Lance transaction.
func NewTransaction ¶
func NewTransaction(readVersion uint64, operation *Operation, opts ...TransactionOption) (*Transaction, error)
NewTransaction constructs a transaction from an operation. The operation is cloned and remains usable. Without WithTransactionUUID, native generates one.
func ParseTransaction ¶
func ParseTransaction(data []byte) (*Transaction, error)
ParseTransaction parses one complete Lance protobuf transaction.
func WriteUncommitted ¶
func WriteUncommitted(ctx context.Context, uri string, reader array.RecordReader, opts ...WriteOption) (*Transaction, error)
WriteUncommitted writes reader as uncommitted data files and returns the transaction. The caller owns the transaction and is responsible for committing it or leaving the data files orphaned. reader is borrowed exactly as in Write.
func (*Transaction) Close ¶
func (t *Transaction) Close() error
Close releases the transaction. It is idempotent.
func (*Transaction) Commit ¶
func (t *Transaction) Commit(ctx context.Context, uri string, opts ...CommitOption) (*Dataset, error)
Commit publishes the transaction to uri.
func (*Transaction) MarshalBinary ¶
func (t *Transaction) MarshalBinary() ([]byte, error)
MarshalBinary serializes the transaction in Lance protobuf format.
func (*Transaction) Operation ¶
func (t *Transaction) Operation() (*Operation, error)
Operation returns a cloned operation.
func (*Transaction) Properties ¶
func (t *Transaction) Properties() (map[string]string, bool, error)
Properties returns the native property map. The boolean distinguishes an absent map from a present empty map.
func (*Transaction) ReadVersion ¶
func (t *Transaction) ReadVersion() (uint64, error)
ReadVersion returns the transaction's read version. Zero is valid for an uncommitted create.
func (*Transaction) Tag ¶
func (t *Transaction) Tag() (string, bool, error)
Tag returns the optional transaction tag.
func (*Transaction) UUID ¶
func (t *Transaction) UUID() (string, error)
UUID returns the transaction UUID.
type TransactionOption ¶
type TransactionOption func(*transactionConfig) error
TransactionOption configures NewTransaction.
func WithTransactionProperties ¶
func WithTransactionProperties(properties map[string]string) TransactionOption
WithTransactionProperties attaches a property map. Passing nil records a present, empty map.
func WithTransactionTag ¶
func WithTransactionTag(tag string) TransactionOption
WithTransactionTag records an optional transaction tag.
func WithTransactionUUID ¶
func WithTransactionUUID(uuid string) TransactionOption
WithTransactionUUID sets an explicit transaction UUID.
type UpdateColumnsResult ¶
type UpdateColumnsResult struct {
// contains filtered or unexported fields
}
UpdateColumnsResult is the owned result of Fragment.UpdateColumns.
func (*UpdateColumnsResult) Close ¶
func (r *UpdateColumnsResult) Close() error
Close releases the result. It is idempotent. The detached fragment returned by Fragment remains independently owned.
func (*UpdateColumnsResult) FieldsModified ¶
func (r *UpdateColumnsResult) FieldsModified() ([]uint32, error)
FieldsModified returns the native modified field IDs.
func (*UpdateColumnsResult) Fragment ¶
func (r *UpdateColumnsResult) Fragment() (*Fragment, error)
Fragment returns detached updated fragment metadata.
func (*UpdateColumnsResult) MatchedOffsets ¶
func (r *UpdateColumnsResult) MatchedOffsets() ([]uint32, error)
MatchedOffsets returns an owned copy of the matched physical row offsets in native order.
type UpdateOperationOptions ¶
type UpdateOperationOptions struct {
// UpdatedFragments are the complete updated fragment metadata entries.
UpdatedFragments []*Fragment
// FieldsModified lists the physical field IDs changed by the update.
FieldsModified []uint32
// UpdatedFragmentOffsets optionally restricts matched rows by fragment ID
// and physical offset. A nil value is the native None value; a non-nil
// value with no entries is the native Some(empty) value.
UpdatedFragmentOffsets *UpdatedFragmentOffsets
}
UpdateOperationOptions is the supported subset of native Update options for publishing fragment update results. Native Update also supports removed and new fragments, compacted SSTables, preservation bitmaps, a caller-selected update mode and an inserted-rows filter; those are not exposed as options. UpdateColumns pins the native RewriteColumns mode and produces none of the other fields.
type UpdatedFragmentOffsets ¶
type UpdatedFragmentOffsets struct {
// contains filtered or unexported fields
}
UpdatedFragmentOffsets is an owned native mapping from fragment ID to physical row offsets. Native owns duplicate handling and the underlying bitmap representation; entry iteration order is unspecified.
func NewUpdatedFragmentOffsets ¶
func NewUpdatedFragmentOffsets(entries []FragmentOffsets) (*UpdatedFragmentOffsets, error)
NewUpdatedFragmentOffsets copies entries into an owned native value. Pass a nil slice to create the explicit empty value. Omit the value from NewUpdateOperation to represent the native absent value.
func (*UpdatedFragmentOffsets) Clone ¶
func (o *UpdatedFragmentOffsets) Clone() (*UpdatedFragmentOffsets, error)
Clone returns an independently owned copy.
func (*UpdatedFragmentOffsets) Close ¶
func (o *UpdatedFragmentOffsets) Close() error
Close releases the offsets. It is idempotent.
func (*UpdatedFragmentOffsets) Count ¶
func (o *UpdatedFragmentOffsets) Count() (uint64, error)
Count returns the number of fragment entries.
func (*UpdatedFragmentOffsets) Entries ¶
func (o *UpdatedFragmentOffsets) Entries() ([]FragmentOffsets, error)
Entries returns every fragment entry in native iteration order.
func (*UpdatedFragmentOffsets) FragmentIDAt ¶
func (o *UpdatedFragmentOffsets) FragmentIDAt(index uint64) (uint64, error)
FragmentIDAt returns the fragment ID of one entry.
type Vector ¶
type Vector struct {
// contains filtered or unexported fields
}
Vector is an immutable, owned query vector buffer. Construct it with NewFloat32Vector, NewFloat16Vector, NewFloat64Vector, NewUInt8Vector, or NewInt8Vector. The constructors copy their input, so the caller may reuse or mutate the source slice immediately after the call.
func NewFloat16Vector ¶
NewFloat16Vector returns a float16 query vector. Each value is an IEEE 754 binary16 bit pattern.
func NewFloat32Vector ¶
NewFloat32Vector returns a float32 query vector.
func NewFloat64Vector ¶
NewFloat64Vector returns a float64 query vector.
func NewInt8Vector ¶
NewInt8Vector returns a signed 8-bit integer query vector.
func NewUInt8Vector ¶
NewUInt8Vector returns an unsigned 8-bit integer query vector.
func (Vector) DataType ¶
func (v Vector) DataType() VectorDataType
DataType returns the vector element type.
type VectorDataType ¶
type VectorDataType int32
VectorDataType identifies the element representation of a query vector.
The values of a Vector for a given data type are interpreted as follows: Float16 uses IEEE 754 binary16 bit patterns, and UInt8 and Int8 use unsigned and signed 8-bit integers respectively. Hamming distance is only defined for UInt8 vectors.
const ( // VectorDataTypeFloat32 uses 32-bit IEEE 754 floating-point values. VectorDataTypeFloat32 VectorDataType = 0 // VectorDataTypeFloat16 uses 16-bit IEEE 754 binary16 bit patterns. VectorDataTypeFloat16 VectorDataType = 1 // VectorDataTypeFloat64 uses 64-bit IEEE 754 floating-point values. VectorDataTypeFloat64 VectorDataType = 2 // VectorDataTypeUInt8 uses unsigned 8-bit integer values. VectorDataTypeUInt8 VectorDataType = 3 // VectorDataTypeInt8 uses signed 8-bit integer values. VectorDataTypeInt8 VectorDataType = 4 )
type VectorMetric ¶
type VectorMetric = Metric
VectorMetric selects the distance metric used for vector search. The dataset or index metric is used when it is not set explicitly.
const ( // VectorMetricL2 uses squared Euclidean distance. VectorMetricL2 VectorMetric = MetricL2 // VectorMetricCosine uses cosine distance. VectorMetricCosine VectorMetric = MetricCosine // VectorMetricDot uses dot-product distance. VectorMetricDot VectorMetric = MetricDot // VectorMetricHamming uses Hamming distance for UInt8 vectors. VectorMetricHamming VectorMetric = MetricHamming )
type WriteFragmentProgress ¶
WriteFragmentProgress is one begin- or complete-fragment progress event.
type WriteOption ¶
type WriteOption func(*writeConfig) error
WriteOption configures Write, Create, Append and Overwrite.
func WithAutoCleanup ¶
func WithAutoCleanup(params AutoCleanupParams) WriteOption
WithAutoCleanup configures old-version cleanup for create writes.
func WithFragmentSchema ¶
func WithFragmentSchema(schema *NativeSchema) WriteOption
WithFragmentSchema sets the schema used by CreateFragment and WriteFragments. The native schema preserves field IDs that are not represented in an Arrow schema. It is invalid for Write, Create, Append, Overwrite and WriteUncommitted or WriteToNamespace.
func WithSkipAutoCleanup ¶
func WithSkipAutoCleanup(skip bool) WriteOption
WithSkipAutoCleanup disables native old-version cleanup.
func WithWriteBranch ¶
func WithWriteBranch(branch string) WriteOption
WithWriteBranch selects a table branch for WriteToNamespace. It is invalid for URI writes.
func WithWriteCommitLock ¶
func WithWriteCommitLock(lock *CommitLock) WriteOption
WithWriteCommitLock routes ordinary and uncommitted writes through an external commit lock. It is not supported by namespace writes or fragment creation.
func WithWriteMode ¶
func WithWriteMode(mode WriteMode) WriteOption
WithWriteMode selects the write mode.
func WithWriteParams ¶
func WithWriteParams(params WriteParams) WriteOption
WithWriteParams sets native write parameters.
func WithWriteSession ¶
func WithWriteSession(session *Session) WriteOption
WithWriteSession pins the write to a session. It is valid for ordinary writes, WriteUncommitted, WriteToNamespace, CreateFragment and WriteFragments.
func WithWriteStorageOptions ¶
func WithWriteStorageOptions(options map[string]string) WriteOption
WithWriteStorageOptions passes object-store options to the write.
func WithWriteStorageOptionsAccessor ¶
func WithWriteStorageOptionsAccessor(accessor *StorageOptionsAccessor) WriteOption
WithWriteStorageOptionsAccessor supplies a reusable dynamic accessor for a dataset write. It is mutually exclusive with WithWriteStorageOptions.
func WithWriteTransactionProperties ¶
func WithWriteTransactionProperties(properties map[string]string) WriteOption
WithWriteTransactionProperties attaches properties to the transaction created by an ordinary write or WriteUncommitted. Passing nil records a present, empty map.
type WriteParams ¶
type WriteParams struct {
MaxRowsPerFile uint64
MaxRowsPerGroup uint64
MaxBytesPerFile uint64
DataStorageVersion string
EnableStableRowIDs bool
EnableV2ManifestPaths *bool
InitialBases []BasePath
TargetBases []uint32
TargetBaseNamesOrPaths []string
TargetAllBases *bool
BaseStoreParams map[string]BaseStoreParams
AllowExternalBlobOutsideBases bool
ExternalBlobMode ExternalBlobMode
BlobPackFileSizeThreshold *uint64
FileWriterOptions *DatasetFileWriterOptions
Progress *WriteProgress
}
WriteParams controls file layout and stable row IDs. Zero values select native Lance defaults.
The native-only fields below are routed through the owned native write options handle. Entry points that still pass the older static write options structure, including namespace writes and fragment creation, reject EnableV2ManifestPaths, InitialBases, TargetBases, TargetBaseNamesOrPaths, TargetAllBases, BaseStoreParams, AllowExternalBlobOutsideBases, ExternalBlobMode, BlobPackFileSizeThreshold, FileWriterOptions and Progress instead of silently dropping them. A commit lock is supplied separately with WithWriteCommitLock and does not change the presence semantics of these fields.
type WriteProgress ¶
type WriteProgress struct {
BeginFragment func(WriteFragmentProgress) error
CompleteFragment func(WriteFragmentProgress) error
WriteStats func(WriteStats)
}
WriteProgress receives native write callbacks. Callbacks run on native worker threads and must be safe for concurrent use.
type WriteStats ¶
WriteStats is a native cumulative write-statistics snapshot.
Source Files
¶
- blob.go
- blob_descriptor.go
- blob_field.go
- blob_read.go
- branches.go
- commit_lock.go
- dataset.go
- dataset_base_paths.go
- dataset_batch_udf.go
- dataset_cleanup.go
- dataset_compaction.go
- dataset_fts.go
- dataset_index.go
- dataset_index_metadata.go
- dataset_index_segment.go
- dataset_index_train.go
- dataset_metadata.go
- dataset_mutation.go
- dataset_mutation_options.go
- dataset_optimize.go
- dataset_schema.go
- dataset_write.go
- dataset_write_native.go
- errors.go
- fragment.go
- fragment_create.go
- fragment_mutation.go
- namespace.go
- operation_mutation.go
- options.go
- refs.go
- resource.go
- scan_fts.go
- scanner.go
- session.go
- storage_options.go
- tags.go
- transaction.go
- vector.go
- versions.go
Directories
¶
| Path | Synopsis |
|---|---|
|
artifacts
|
|
|
darwin-amd64
module
|
|
|
darwin-arm64
module
|
|
|
linux-amd64
module
|
|
|
linux-arm64
module
|
|
|
windows-amd64
module
|
|
|
windows-arm64
module
|
|
|
benchmarks
|
|
|
soak
command
|
|
|
Package compaction contains the option, metric, and transport values used by dataset compaction.
|
Package compaction contains the option, metric, and transport values used by dataset compaction. |
|
compat
|
|
|
go
command
Command lance-compat-go is the Go producer/consumer for the Phase 5 cross-language Lance compatibility fixture.
|
Command lance-compat-go is the Go producer/consumer for the Phase 5 cross-language Lance compatibility fixture. |
|
examples
|
|
|
cloud-refresh
command
|
|
|
roundtrip
command
Command roundtrip creates, reopens and scans a temporary Lance dataset.
|
Command roundtrip creates, reopens and scans a temporary Lance dataset. |
|
Package file provides the raw Lance file API.
|
Package file provides the raw Lance file API. |
|
Package index defines the public vector and scalar index options, enums and metadata values used by the lance dataset package.
|
Package index defines the public vector and scalar index options, enums and metadata values used by the lance dataset package. |
|
internal
|
|
|
accessor
Package accessor owns a reusable native storage-options accessor and the dynamic Go provider behind it.
|
Package accessor owns a reusable native storage-options accessor and the dynamic Go provider behind it. |
|
bundled
Package bundled activates the compiled-in Lance native library selectors.
|
Package bundled activates the compiled-in Lance native library selectors. |
|
cdata
Package cdata implements the Arrow C Data Interface and the Arrow C Stream Interface in pure Go.
|
Package cdata implements the Arrow C Data Interface and the Arrow C Stream Interface in pure Go. |
|
ffi
Package ffi binds the subset of the lance-c C ABI used by lance-go.
|
Package ffi binds the subset of the lance-c C ABI used by lance-go. |
|
handle
Package handle stores Go objects behind integer identifiers that can be passed through foreign-function boundaries.
|
Package handle stores Go objects behind integer identifiers that can be passed through foreign-function boundaries. |
|
lifetime
Package lifetime tracks in-flight operations on an owned native handle.
|
Package lifetime tracks in-flight operations on an owned native handle. |
|
loader
Package loader discovers and loads the official lance-c shared library without cgo.
|
Package loader discovers and loads the official lance-c shared library without cgo. |
|
nshandle
Package nshandle is the narrow ownership boundary between the public namespace package, the root lance package, and the native FFI.
|
Package nshandle is the narrow ownership boundary between the public namespace package, the root lance package, and the native FFI. |
|
session
Package session owns shared native Lance session lifetimes.
|
Package session owns shared native Lance session lifetimes. |
|
Package namespace provides typed access to Lance namespaces through the native lance C API.
|
Package namespace provides typed access to Lance namespaces through the native lance C API. |
|
Command check-abi-symbols verifies the purego bindings in internal/ffi against the lance-c C header from the commit pinned in that package.
|
Command check-abi-symbols verifies the purego bindings in internal/ffi against the lance-c C header from the commit pinned in that package. |
|
internal/artifactgen
Package artifactgen builds a data-only Go module and a platform selector from an already-built lance-c shared library and its matching header.
|
Package artifactgen builds a data-only Go module and a platform selector from an already-built lance-c shared library and its matching header. |
|
public-consumer/namespace-consumer
command
Command namespace-consumer opens a native directory namespace without importing the root lance-go package.
|
Command namespace-consumer opens a native directory namespace without importing the root lance-go package. |
|
public-consumer/windows-consumer
command
Command windows-consumer loads the public namespace API and reports the native module that the process actually mapped.
|
Command windows-consumer loads the public namespace API and reports the native module that the process actually mapped. |
|
sync-lance-c-gen
command
|
|
|
verify-lance-c-artifact
command
Command verify-lance-c-artifact verifies one generated lance-c artifact module against the native library it was packaged from and the pin in internal/ffi/upstream.go.
|
Command verify-lance-c-artifact verifies one generated lance-c artifact module against the native library it was packaged from and the pin in internal/ffi/upstream.go. |