model

package
v0.0.0-...-bace2a7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 12, 2026 License: Apache-2.0 Imports: 3 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// MetadataPropertyPrefix is the key prefix for XTable sync metadata stored in table properties.
	MetadataPropertyPrefix = "xtable_"
	// KeyLastInstantSynced is the property key for the last synced instant.
	KeyLastInstantSynced = "xtable_last_instant_synced"
	// KeySourceFormat is the property key for the source table format.
	KeySourceFormat = "xtable_source_format"
)

Variables

This section is empty.

Functions

This section is empty.

Types

type ColumnStat 

type ColumnStat struct {
	// Field refers to the schema field these statistics describe.
	Field *Field `json:"field"`
	// Range is the minimum and maximum value bounds.
	Range *Range `json:"range,omitempty"`
	// NumNulls is the count of null values in this column.
	NumNulls int64 `json:"numNulls"`
	// NumNaNs is the count of NaN (Not a Number) values for floating-point columns.
	NumNaNs int64 `json:"numNaNs"`
	// TotalValues is the total count of values in this column.
	TotalValues int64 `json:"totalValues"`
}

ColumnStat represents summary statistics for a column within a data file.

type DataFile 

type DataFile struct {
	// PhysicalPath is the fully qualified URI or relative path to the physical data file.
	PhysicalPath string `json:"physicalPath"`
	// FileFormat is the physical storage format (e.g. APACHE_PARQUET, APACHE_ORC).
	FileFormat FileFormat `json:"fileFormat"`
	// FileSizeBytes is the size of the data file in bytes.
	FileSizeBytes int64 `json:"fileSizeBytes"`
	// RecordCount is the total number of records contained in this file.
	RecordCount int64 `json:"recordCount"`
	// PartitionValues specifies the partition values for this file.
	PartitionValues []*PartitionValue `json:"partitionValues,omitempty"`
	// ColumnStats holds column-level min/max, null count, and NaN statistics.
	ColumnStats []*ColumnStat `json:"columnStats,omitempty"`
	// LastModified is the file last modified timestamp in milliseconds since epoch.
	LastModified int64 `json:"lastModified"`
	// DeletionVector optionally points to row deletion vector metadata.
	DeletionVector *DeletionVector `json:"deletionVector,omitempty"`
}

DataFile represents a physical data file in the table.

type DataLayoutStrategy 

type DataLayoutStrategy string

DataLayoutStrategy describes how data files are structured on physical storage. 

const (
	// DataLayoutDirHierarchical represents hierarchical slash-separated paths (e.g. /2026/08/12/).
	DataLayoutDirHierarchical DataLayoutStrategy = "DIR_HIERARCHICAL"
	// DataLayoutHiveStyle represents standard Hive key=value partition paths (e.g. /year=2026/month=08/).
	DataLayoutHiveStyle DataLayoutStrategy = "HIVE_STYLE"
	// DataLayoutFlat represents unpartitioned flat storage under table data path.
	DataLayoutFlat DataLayoutStrategy = "FLAT"
)

type DeletionVector 

type DeletionVector struct {
	// StoragePath is the relative or absolute path to the deletion vector file if stored externally.
	StoragePath string `json:"storagePath,omitempty"`
	// Offset is the byte offset where the deletion vector bitmap starts.
	Offset int64 `json:"offset,omitempty"`
	// SizeInBytes is the byte length of the deletion vector.
	SizeInBytes int64 `json:"sizeInBytes"`
	// Cardinality is the number of deleted records marked by this vector.
	Cardinality int64 `json:"cardinality"`
	// InlineBytes contains raw bitmap bytes if stored inline within metadata.
	InlineBytes []byte `json:"inlineBytes,omitempty"`
}

DeletionVector represents row deletion metadata associated with a data file (e.g. Delta/Iceberg).

type Field 

type Field struct {
	// Name is the field identifier.
	Name string `json:"name"`
	// ParentPath is the dot-delimited path to the parent (empty for top-level fields).
	ParentPath string `json:"parentPath,omitempty"`
	// Schema is the data type and nested structure of this field.
	Schema *Schema `json:"schema"`
	// FieldID is the optional unique integer field identifier (critical for Iceberg).
	FieldID *int `json:"fieldId,omitempty"`
	// DefaultValue represents a default value if defined.
	DefaultValue any `json:"defaultValue,omitempty"`
}

Field represents a single named field in an internal schema hierarchy.

func (*Field) Path 

func (f *Field) Path() string

Path returns the full dot-delimited path of the field (e.g. "address.city").

type FileFormat 

type FileFormat string

FileFormat represents the physical data file format. 

const (
	FileFormatParquet FileFormat = "APACHE_PARQUET"
	FileFormatORC     FileFormat = "APACHE_ORC"
	FileFormatAvro    FileFormat = "APACHE_AVRO"
)

Supported physical data file formats.

type FilesDiff 

type FilesDiff struct {
	// FilesAdded is the list of new or updated data files added in this commit.
	FilesAdded []*DataFile `json:"filesAdded,omitempty"`
	// FilesRemoved is the list of data files removed or compacted away in this commit.
	FilesRemoved []*DataFile `json:"filesRemoved,omitempty"`
}

FilesDiff captures the addition and removal of data files between two table states or commits.

func DiffFiles 

func DiffFiles(oldFiles, newFiles []*DataFile) *FilesDiff

DiffFiles computes the difference between an old set and a new set of data files.

func NewFilesDiff 

func NewFilesDiff(added, removed []*DataFile) *FilesDiff

NewFilesDiff creates a new FilesDiff struct.

func (*FilesDiff) HasChanges 

func (d *FilesDiff) HasChanges() bool

HasChanges returns true if there are any added or removed files.

type IncrementalTableChanges 

type IncrementalTableChanges struct {
	// TableChanges contains the sequential list of commits to be replayed on target formats.
	TableChanges []*TableChange `json:"tableChanges"`
	// CurrentTable is the latest metadata state of the source table.
	CurrentTable *Table `json:"currentTable"`
}

IncrementalTableChanges represents an ordered sequence of commits for incremental synchronization.

type MetadataKey 

type MetadataKey string

MetadataKey represents keys for type-specific metadata (e.g. decimal precision, time unit). 

const (
	MetadataKeyDecimalScale       MetadataKey = "DECIMAL_SCALE"
	MetadataKeyDecimalPrecision   MetadataKey = "DECIMAL_PRECISION"
	MetadataKeyEnumValues         MetadataKey = "ENUM_VALUES"
	MetadataKeyFixedBytesSize     MetadataKey = "FIXED_BYTES_SIZE"
	MetadataKeyTimestampPrecision MetadataKey = "TIMESTAMP_PRECISION"
)

Type-specific metadata keys.

type MetadataValue 

type MetadataValue string

MetadataValue represents values for metadata configurations. 

const (
	MetadataValueMicros MetadataValue = "MICROS"
	MetadataValueMillis MetadataValue = "MILLIS"
	MetadataValueNanos  MetadataValue = "NANOS"
)

Time-unit metadata values.

type PartitionField 

type PartitionField struct {
	// SourceField is the reference to the table field used for partitioning.
	SourceField *Field `json:"sourceField"`
	// TransformType describes how the partition value is generated from the field value.
	TransformType PartitionTransformType `json:"transformType"`
	// Format is an optional date/time pattern (e.g., yyyy-MM-dd) when transform type is temporal.
	Format string `json:"format,omitempty"`
	// CustomPartitionName is an optional custom partition column name.
	CustomPartitionName string `json:"customPartitionName,omitempty"`
}

PartitionField represents a partition field specification.

type PartitionFileGroup 

type PartitionFileGroup struct {
	// PartitionPath is the directory partition path (e.g. "date=2026-08-12").
	PartitionPath string `json:"partitionPath"`
	// Files is the list of data files belonging to this partition.
	Files []*DataFile `json:"files"`
}

PartitionFileGroup groups data files by partition path.

type PartitionTransformType 

type PartitionTransformType string

PartitionTransformType represents the transform applied on a source column to produce partition values. 

const (
	PartitionTransformValue PartitionTransformType = "VALUE"
	PartitionTransformYear  PartitionTransformType = "YEAR"
	PartitionTransformMonth PartitionTransformType = "MONTH"
	PartitionTransformDay   PartitionTransformType = "DAY"
	PartitionTransformHour  PartitionTransformType = "HOUR"
)

Supported partition transforms.

type PartitionValue 

type PartitionValue struct {
	// PartitionField is the partition specification field.
	PartitionField *PartitionField `json:"partitionField"`
	// Range represents the partition value range (exact value for standard partitioning).
	Range *Range `json:"range"`
}

PartitionValue associates a partition field with its specific value range for a data file.

type Range 

type Range struct {
	// MinValue is the minimum value in the range.
	MinValue any `json:"minValue,omitempty"`
	// MaxValue is the maximum value in the range.
	MaxValue any `json:"maxValue,omitempty"`
}

Range represents the lower and upper bounds of a column's values.

func NewRange 

func NewRange(minVal, maxVal any) *Range

NewRange creates a new Range with min and max values.

func NewScalarRange 

func NewScalarRange(val any) *Range

NewScalarRange creates a Range where min and max are equal (e.g. for exact partition values).

type Schema 

type Schema struct {
	// Name of this schema definition (optional for anonymous types).
	Name string `json:"name,omitempty"`
	// DataType is the canonical data type.
	DataType Type `json:"dataType"`
	// Comment is a user-readable description of this field/schema.
	Comment string `json:"comment,omitempty"`
	// IsNullable indicates if values of this field can be null.
	IsNullable bool `json:"isNullable"`
	// Fields contains the list of child fields for RECORD / STRUCT types.
	Fields []*Field `json:"fields,omitempty"`
	// ElementSchema contains the item schema when DataType is LIST.
	ElementSchema *Field `json:"elementSchema,omitempty"`
	// KeySchema and ValueSchema contain key and value field schemas when DataType is MAP.
	KeySchema   *Field `json:"keySchema,omitempty"`
	ValueSchema *Field `json:"valueSchema,omitempty"`
	// RecordKeyFields lists the primary key / record key fields for the table.
	RecordKeyFields []*Field `json:"recordKeyFields,omitempty"`
	// Metadata holds type-specific metadata (e.g. DECIMAL_PRECISION, DECIMAL_SCALE).
	Metadata map[MetadataKey]any `json:"metadata,omitempty"`
}

Schema represents a type definition in XTable's internal schema model.

func NewDecimalSchema 

func NewDecimalSchema(precision, scale int, nullable bool) *Schema

NewDecimalSchema creates an internal schema for a decimal type with precision and scale.

func NewPrimitiveSchema 

func NewPrimitiveSchema(t Type, nullable bool) *Schema

NewPrimitiveSchema creates an internal schema for a primitive scalar type.

func NewRecordSchema 

func NewRecordSchema(name string, fields []*Field, nullable bool) *Schema

NewRecordSchema creates a composite struct/record schema with child fields.

func (*Schema) AllFields 

func (s *Schema) AllFields() []*Field

AllFields performs a level-order traversal and returns all top-level and nested fields.

func (*Schema) FieldByPath 

func (s *Schema) FieldByPath(path string) *Field

FieldByPath searches for a field matching the dot-delimited path.

Matching prefers an exact, case-sensitive match at each level and falls back to a case-insensitive one only when no exact match exists. The fallback is deliberate: format adapters pass partition column names taken from format metadata, which does not always agree with the schema on case. Preferring the exact match first means a schema holding both "Name" and "name" resolves predictably rather than returning whichever field happened to come first.

type Snapshot 

type Snapshot struct {
	// Table is the metadata descriptor of the table at this snapshot instant.
	Table *Table `json:"table"`
	// PartitionedDataFiles groups data files by partition path.
	PartitionedDataFiles []*PartitionFileGroup `json:"partitionedDataFiles,omitempty"`
	// DataFiles holds a flat list of all active data files in this snapshot.
	DataFiles []*DataFile `json:"dataFiles,omitempty"`
	// SourceIdentifier is the format-specific commit version or timestamp string.
	SourceIdentifier string `json:"sourceIdentifier"`
}

Snapshot captures the complete state of a table at a specific point in time or commit.

func (*Snapshot) AllDataFiles 

func (s *Snapshot) AllDataFiles() []*DataFile

AllDataFiles returns a consolidated slice of all active data files in this snapshot.

type Table 

type Table struct {
	// Name is the logical table name.
	Name string `json:"name"`
	// TableFormat indicates the underlying format (HUDI, ICEBERG, DELTA, PAIMON, PARQUET).
	TableFormat TableFormat `json:"tableFormat"`
	// ReadSchema is the current canonical schema for reading data from this table.
	ReadSchema *Schema `json:"readSchema"`
	// LayoutStrategy is the data directory layout strategy.
	LayoutStrategy DataLayoutStrategy `json:"layoutStrategy,omitempty"`
	// BasePath is the root directory path containing table metadata and data.
	BasePath string `json:"basePath"`
	// DataPath is the path containing data files (defaults to BasePath if empty).
	DataPath string `json:"dataPath,omitempty"`
	// PartitioningFields contains the ordered list of partition field definitions.
	PartitioningFields []*PartitionField `json:"partitioningFields,omitempty"`
	// LatestCommitTime is the timestamp of the latest write/commit in milliseconds since epoch.
	LatestCommitTime int64 `json:"latestCommitTime"`
	// LatestMetadataPath is the path to the latest metadata descriptor file.
	LatestMetadataPath string `json:"latestMetadataPath,omitempty"`
}

Table represents the canonical reference and current metadata state of a lakehouse table.

func (*Table) GetDataPath 

func (t *Table) GetDataPath() string

GetDataPath returns the effective data directory path (falling back to BasePath).

func (*Table) IsPartitioned 

func (t *Table) IsPartitioned() bool

IsPartitioned returns true if the table has one or more partitioning fields.

type TableChange 

type TableChange struct {
	// FilesDiff records the data files added and removed in this change.
	FilesDiff *FilesDiff `json:"filesDiff"`
	// TableAsOfChange is the table metadata (schema, partitioning) as of this commit.
	TableAsOfChange *Table `json:"tableAsOfChange"`
	// SourceIdentifier is the unique commit version/instant identifier (e.g. Delta version "5", Hudi timestamp).
	SourceIdentifier string `json:"sourceIdentifier"`
	// CommitTime is the commit timestamp in milliseconds since epoch.
	CommitTime int64 `json:"commitTime"`
}

TableChange captures the delta modifications applied in a single commit or instant on the source table.

type TableFormat 

type TableFormat string

TableFormat represents standard lakehouse table format identifiers. 

const (
	TableFormatHudi    TableFormat = "HUDI"
	TableFormatIceberg TableFormat = "ICEBERG"
	TableFormatDelta   TableFormat = "DELTA"
	TableFormatPaimon  TableFormat = "PAIMON"
	TableFormatParquet TableFormat = "PARQUET"
)

Supported lakehouse table formats.

func ParseTableFormat 

func ParseTableFormat(s string) (TableFormat, error)

ParseTableFormat converts a string to TableFormat.

func SupportedTableFormats 

func SupportedTableFormats() []TableFormat

SupportedTableFormats returns the list of all supported table formats.

type TableSyncMetadata 

type TableSyncMetadata struct {
	// LastInstantSynced is the timestamp (epoch millis) of the latest successfully synced commit.
	LastInstantSynced int64 `json:"lastInstantSynced"`
	// InstantsToConsiderForNextSync tracks any pending or intermediate commits required for incremental continuity.
	InstantsToConsiderForNextSync []int64 `json:"instantsToConsiderForNextSync,omitempty"`
	// SourceFormat is the source format that was translated into this table.
	SourceFormat TableFormat `json:"sourceFormat,omitempty"`
	// TargetFormat is the format of the table storing this sync metadata.
	TargetFormat TableFormat `json:"targetFormat,omitempty"`
	// CustomProperties stores additional provider-specific sync attributes.
	CustomProperties map[string]string `json:"customProperties,omitempty"`
}

TableSyncMetadata captures synchronization state embedded inside target table metadata/properties.

type Type 

type Type string

Type represents the canonical data type in XTable's internal type system. 

const (
	TypeRecord       Type = "RECORD"
	TypeEnum         Type = "ENUM"
	TypeList         Type = "LIST"
	TypeMap          Type = "MAP"
	TypeUnion        Type = "UNION"
	TypeUUID         Type = "UUID"
	TypeFixed        Type = "FIXED"
	TypeString       Type = "STRING"
	TypeBytes        Type = "BYTES"
	TypeInt          Type = "INT"
	TypeLong         Type = "LONG"
	TypeFloat        Type = "FLOAT"
	TypeDouble       Type = "DOUBLE"
	TypeBoolean      Type = "BOOLEAN"
	TypeNull         Type = "NULL"
	TypeDate         Type = "DATE"
	TypeDecimal      Type = "DECIMAL"
	TypeTimestamp    Type = "TIMESTAMP"
	TypeTimestampNTZ Type = "TIMESTAMP_NTZ"
)

Canonical data types.

func (Type) IsNonScalar 

func (t Type) IsNonScalar() bool

IsNonScalar returns true if the type is a composite/nested type.

func (Type) String 

func (t Type) String() string

String returns the string representation of Type.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.