meta

Documentation

Index

• Constants
• Variables
• func DataPath(chunk, file int) string
• func EpisodeChunk(episodeIndex, chunksSize int) int
• func EpisodesMetaPath(chunk, file int) string
• func FileSizeMB(path string) (float64, error)
• func FormatChunkFile(chunk, file int) string
• func FormatPathTemplate(template string, episodeIndex, chunkIndex, fileIndex int, videoKey string) string
• func ParquetUncompressedSizeMB(path string) (float64, error)
• func UpdateChunkFileIndices(chunkIdx, fileIdx, chunksSize int) (int, int)
• func UpdateVideoFeaturesInfo(ctx context.Context, info *DatasetInfo, root string, locator video.Locator) error
• func V21DataPath(episode int) string
• func V21DataPathFromInfo(info DatasetInfo, episodeIndex int) string
• func V21VideoPath(videoKey string, episode int) string
• func V21VideoPathFromInfo(info DatasetInfo, videoKey string, episodeIndex int) string
• func VideoPath(videoKey string, chunk, file int) string
• func WriteInfo(root string, info DatasetInfo) error
• func WriteStats(root string, stats map[string]map[string]any) error
• type DatasetInfo
  • func LoadInfo(root string) (DatasetInfo, error)
  • func NewDatasetInfo(codebase string, fps int, features map[string]FeatureSpec, useVideos bool, ...) DatasetInfo
• type FeatureSpec
  • func (f FeatureSpec) Equal(other FeatureSpec) bool
  • func (f FeatureSpec) MarshalJSON() ([]byte, error)
  • func (f *FeatureSpec) UnmarshalJSON(data []byte) error

Constants

const (
	CodebaseV21 = "v2.1"
	CodebaseV30 = "v3.0"

	// Official path templates (lerobot v0.3.3 / v0.5.1 datasets.utils).
	V21DefaultDataPathTemplate  = "data/chunk-{episode_chunk:03d}/episode_{episode_index:06d}.parquet"
	V21DefaultVideoPathTemplate = "videos/chunk-{episode_chunk:03d}/{video_key}/episode_{episode_index:06d}.mp4"
	V30DefaultDataPathTemplate  = "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet"
	V30DefaultVideoPathTemplate = "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4"

	DefaultChunkSize         = 1000
	DefaultDataFileSizeInMB  = 100
	DefaultVideoFileSizeInMB = 200
	ChunkFilePattern         = "chunk-%03d/file-%03d"
	InfoPath                 = "meta/info.json"
	StatsPath                = "meta/stats.json"
	DefaultTasksPath         = "meta/tasks.parquet"
	EpisodesDir              = "meta/episodes"
	DataDir                  = "data"
	VideoDir                 = "videos"
	LegacyEpisodesPath       = "meta/episodes.jsonl"
	LegacyEpisodesStatsPath  = "meta/episodes_stats.jsonl"
	LegacyTasksPath          = "meta/tasks.jsonl"
	DefaultImagePath         = "images/%s/episode-%06d/frame-%06d.png"
)

Variables

var DefaultFeatures = map[string]FeatureSpec{
	"timestamp":     {DType: "float32", Shape: []int{1}},
	"frame_index":   {DType: "int64", Shape: []int{1}},
	"episode_index": {DType: "int64", Shape: []int{1}},
	"index":         {DType: "int64", Shape: []int{1}},
	"task_index":    {DType: "int64", Shape: []int{1}},
}

DefaultFeatures are auto-injected columns (lerobot.utils.constants.DEFAULT_FEATURES).

Functions

func DataPath

func DataPath(chunk, file int) string

func EpisodeChunk

func EpisodeChunk(episodeIndex, chunksSize int) int

EpisodeChunk returns the chunk index for a v2.1 per-episode layout.

func EpisodesMetaPath

func EpisodesMetaPath(chunk, file int) string

func FileSizeMB

func FileSizeMB(path string) (float64, error)

FileSizeMB returns on-disk file size in megabytes.

func FormatChunkFile

func FormatChunkFile(chunk, file int) string

func FormatPathTemplate

func FormatPathTemplate(template string, episodeIndex, chunkIndex, fileIndex int, videoKey string) string

FormatPathTemplate expands LeRobot path templates from info.json.

func ParquetUncompressedSizeMB

func ParquetUncompressedSizeMB(path string) (float64, error)

ParquetUncompressedSizeMB mirrors io_utils.get_parquet_file_size_in_mb (uncompressed bytes).

func UpdateChunkFileIndices

func UpdateChunkFileIndices(chunkIdx, fileIdx, chunksSize int) (int, int)

UpdateChunkFileIndices mirrors lerobot.datasets.utils.update_chunk_file_indices.

func UpdateVideoFeaturesInfo

func UpdateVideoFeaturesInfo(ctx context.Context, info *DatasetInfo, root string, locator video.Locator) error

UpdateVideoFeaturesInfo fills features[].info from the first on-disk video file, matching dataset_metadata.update_video_info (lerobot v0.5.1).

func V21DataPath

func V21DataPath(episode int) string

func V21DataPathFromInfo

func V21DataPathFromInfo(info DatasetInfo, episodeIndex int) string

V21DataPathFromInfo resolves the data parquet path for an episode using info.DataPath.

func V21VideoPath

func V21VideoPath(videoKey string, episode int) string

func V21VideoPathFromInfo

func V21VideoPathFromInfo(info DatasetInfo, videoKey string, episodeIndex int) string

V21VideoPathFromInfo resolves a video path for an episode and camera key.

func VideoPath

func VideoPath(videoKey string, chunk, file int) string

func WriteInfo

func WriteInfo(root string, info DatasetInfo) error

func WriteStats

func WriteStats(root string, stats map[string]map[string]any) error

Types

type DatasetInfo

type DatasetInfo struct {
	CodebaseVersion    string                 `json:"codebase_version"`
	FPS                int                    `json:"fps"`
	Features           map[string]FeatureSpec `json:"features"`
	TotalEpisodes      int                    `json:"total_episodes"`
	TotalFrames        int                    `json:"total_frames"`
	TotalTasks         int                    `json:"total_tasks"`
	ChunksSize         int                    `json:"chunks_size"`
	DataFilesSizeInMB  int                    `json:"data_files_size_in_mb"`
	VideoFilesSizeInMB int                    `json:"video_files_size_in_mb"`
	DataPath           string                 `json:"data_path"`
	VideoPath          *string                `json:"video_path,omitempty"`
	RobotType          *string                `json:"robot_type,omitempty"`
	Splits             map[string]string      `json:"splits,omitempty"`
	TotalChunks        *int                   `json:"total_chunks,omitempty"`
	TotalVideos        *int                   `json:"total_videos,omitempty"`
}

func LoadInfo

func LoadInfo(root string) (DatasetInfo, error)

func NewDatasetInfo

func NewDatasetInfo(codebase string, fps int, features map[string]FeatureSpec, useVideos bool, robotType string) DatasetInfo

type FeatureSpec

type FeatureSpec struct {
	DType string         `json:"dtype"`
	Shape []int          `json:"shape"`
	Names interface{}    `json:"names,omitempty"`
	Info  map[string]any `json:"info,omitempty"`
	Extra map[string]any `json:"-"`
}

func (FeatureSpec) Equal

func (f FeatureSpec) Equal(other FeatureSpec) bool

func (FeatureSpec) MarshalJSON

func (f FeatureSpec) MarshalJSON() ([]byte, error)

func (*FeatureSpec) UnmarshalJSON

func (f *FeatureSpec) UnmarshalJSON(data []byte) error