dataset

Documentation

Index

• Constants
• func GetSamples(url string) ([]string, error)
• type DataSource
• type Dataset
• type Manager
  • func New(client clienttypes.ClientI, options *options.LocalControllerOptions) *Manager
  • func (dm *Manager) AddWorkerMessage(message workertypes.MessageContent)
  • func (dm *Manager) Delete(message *clienttypes.Message) error
  • func (dm *Manager) GetDataset(name string) (*Dataset, bool)
  • func (dm *Manager) GetName() string
  • func (dm *Manager) Insert(message *clienttypes.Message) error
  • func (dm *Manager) Start() error

Constants

const (
	// MonitorDataSourceIntervalSeconds is interval time of monitoring data source
	MonitorDataSourceIntervalSeconds = 60
	// KindName is kind of dataset resource
	KindName = "dataset"
	// CSVFormat is commas separated value format with a extra header.
	// It can be used in structured data scenarios.
	CSVFormat = "csv"
	// TXTFormat is line separated format.
	// It can be used in unstructured data scenarios.
	TXTFormat = "txt"
)

Functions

func GetSamples

func GetSamples(url string) ([]string, error)

GetSamples gets samples in a file

Types

type DataSource

type DataSource struct {
	TrainSamples    []string
	NumberOfSamples int
	Header          string
}

DataSource defines config for data source

type Dataset

type Dataset struct {
	*sednav1.Dataset
	DataSource *DataSource `json:"dataSource"`
	Done       chan struct{}
	URLPrefix  string
	Storage    storage.Storage
}

Dataset defines config for dataset

type Manager

type Manager struct {
	Client            clienttypes.ClientI
	DatasetMap        map[string]*Dataset
	VolumeMountPrefix string
}

DatasetManager defines dataset manager

func New

func New(client clienttypes.ClientI, options *options.LocalControllerOptions) *Manager

New creates a dataset manager

func (*Manager) AddWorkerMessage

func (dm *Manager) AddWorkerMessage(message workertypes.MessageContent)

func (*Manager) Delete

func (dm *Manager) Delete(message *clienttypes.Message) error

Delete deletes dataset config in db

func (*Manager) GetDataset

func (dm *Manager) GetDataset(name string) (*Dataset, bool)

GetDatasetChannel gets dataset

func (*Manager) GetName

func (dm *Manager) GetName() string

func (*Manager) Insert

func (dm *Manager) Insert(message *clienttypes.Message) error

Insert inserts dataset to db

func (*Manager) Start

func (dm *Manager) Start() error

Start starts dataset manager