dataset

Documentation

Index

• func DatasetDir(baseDir, namespace, name string) string
• func EnsureDatasetDir(baseDir, namespace, name string) error
• func EnsureLocalDatasetFiles(datasetDir string, d *LocalDataset) (bool, error)
• func ListDatasetsInNamespace(baseDir, namespace string) ([]string, error)
• func ListNamespaces(baseDir string) ([]string, error)
• func ManifestPath(baseDir, namespace, name string) string
• func RemoveDatasetDir(baseDir, namespace, name string) error
• func SaveManifest(baseDir string, d *LocalDataset) error
• type FileEntry
• type LocalDataset
  • func LoadManifest(baseDir, namespace, name string) (*LocalDataset, error)
  • func (d *LocalDataset) FullName() string
• type LocalDatasetFile
• type LocalDatasetOrigin
• type Manager
  • func NewManager(cfg *config.Config) *Manager
  • func (m *Manager) Client() *csghub.Client
  • func (m *Manager) DatasetPath(datasetID string) (string, error)
  • func (m *Manager) Exists(datasetID string) bool
  • func (m *Manager) Get(datasetID string) (*LocalDataset, error)
  • func (m *Manager) GetWithFileEntries(datasetID string) (*LocalDataset, error)
  • func (m *Manager) List() ([]*LocalDataset, error)
  • func (m *Manager) ListFiles(datasetID, subPath string) ([]FileEntry, error)
  • func (m *Manager) Pull(ctx context.Context, datasetID string, progress csghub.SnapshotProgressFunc) (*LocalDataset, error)
  • func (m *Manager) Remove(datasetID string) error

Functions

func DatasetDir

func DatasetDir(baseDir, namespace, name string) string

func EnsureDatasetDir

func EnsureDatasetDir(baseDir, namespace, name string) error

func EnsureLocalDatasetFiles

func EnsureLocalDatasetFiles(datasetDir string, d *LocalDataset) (bool, error)

func ListDatasetsInNamespace

func ListDatasetsInNamespace(baseDir, namespace string) ([]string, error)

func ListNamespaces

func ListNamespaces(baseDir string) ([]string, error)

func ManifestPath

func ManifestPath(baseDir, namespace, name string) string

func RemoveDatasetDir

func RemoveDatasetDir(baseDir, namespace, name string) error

func SaveManifest

func SaveManifest(baseDir string, d *LocalDataset) error

Types

type FileEntry

type FileEntry struct {
	Name       string    `json:"name"`
	Size       int64     `json:"size"`
	IsDir      bool      `json:"is_dir"`
	ModifiedAt time.Time `json:"modified_at"`
}

type LocalDataset

type LocalDataset struct {
	Namespace    string             `json:"namespace"`
	Name         string             `json:"name"`
	Size         int64              `json:"size"`
	Files        []string           `json:"files"`
	FileEntries  []LocalDatasetFile `json:"file_entries,omitempty"`
	DownloadedAt time.Time          `json:"downloaded_at"`
	Origin       LocalDatasetOrigin `json:"origin,omitempty"`
	Description  string             `json:"description,omitempty"`
	License      string             `json:"license,omitempty"`
}

func LoadManifest

func LoadManifest(baseDir, namespace, name string) (*LocalDataset, error)

func (*LocalDataset) FullName

func (d *LocalDataset) FullName() string

type LocalDatasetFile

type LocalDatasetFile struct {
	Path   string `json:"path"`
	Size   int64  `json:"size"`
	SHA256 string `json:"sha256,omitempty"`
	LFS    bool   `json:"lfs,omitempty"`
}

type LocalDatasetOrigin

type LocalDatasetOrigin string

const (
	LocalDatasetOriginUpload      LocalDatasetOrigin = "upload"
	LocalDatasetOriginMarketplace LocalDatasetOrigin = "marketplace"
)

type Manager

type Manager struct {
	// contains filtered or unexported fields
}

func NewManager

func NewManager(cfg *config.Config) *Manager

func (*Manager) Client

func (m *Manager) Client() *csghub.Client

func (*Manager) DatasetPath

func (m *Manager) DatasetPath(datasetID string) (string, error)

func (*Manager) Exists

func (m *Manager) Exists(datasetID string) bool

func (*Manager) Get

func (m *Manager) Get(datasetID string) (*LocalDataset, error)

func (*Manager) GetWithFileEntries

func (m *Manager) GetWithFileEntries(datasetID string) (*LocalDataset, error)

func (*Manager) List

func (m *Manager) List() ([]*LocalDataset, error)

func (*Manager) ListFiles

func (m *Manager) ListFiles(datasetID, subPath string) ([]FileEntry, error)

func (*Manager) Pull

func (m *Manager) Pull(ctx context.Context, datasetID string, progress csghub.SnapshotProgressFunc) (*LocalDataset, error)

func (*Manager) Remove

func (m *Manager) Remove(datasetID string) error