wat

package
v0.0.0-...-fb1c7ab Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 15, 2022 License: MIT Imports: 21 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AwsConfig 

type AwsConfig struct {
	Name                  string `json:"aws_config_name,omitempty"`
	IsPrimary             bool   `json:"is_primary_config"` //where payloads would get stored?
	AWS_ACCESS_KEY_ID     string `json:"aws_access_key_id"`
	AWS_SECRET_ACCESS_KEY string `json:"aws_secret_access_key_id"`
	AWS_REGION            string `json:"aws_region"`
	AWS_BUCKET            string `json:"aws_bucket"`
	S3_MOCK               bool   `json:"aws_mock,omitempty"`             //for testing with minio
	S3_ENDPOINT           string `json:"aws_endpoint,omitempty"`         //for testing with minio
	S3_DISABLE_SSL        bool   `json:"aws_disable_ssl,omitempty"`      //for testing with minio
	S3_FORCE_PATH_STYLE   bool   `json:"aws_force_path_style,omitempty"` //for testing with minio
}

func (AwsConfig) EnvironmentVariables 

func (c AwsConfig) EnvironmentVariables() []string

type BatchCloudProvider 

type BatchCloudProvider struct {
	BatchSession *batch.Batch
	Resources    map[string]provisionedResources
}

func (BatchCloudProvider) Dependencies 

func (b BatchCloudProvider) Dependencies(lm LinkedModelManifest, eventIndex int, dag DirectedAcyclicGraph) ([]*string, error)

func (BatchCloudProvider) ProcessTask 

func (b BatchCloudProvider) ProcessTask(job *Job, eventIndex int, payloadPath string, linkedManifest LinkedModelManifest) error

func (BatchCloudProvider) ProvisionResources 

func (b BatchCloudProvider) ProvisionResources(jobManager JobManager) error

func (BatchCloudProvider) ResourcesForLinkedManifest 

func (b BatchCloudProvider) ResourcesForLinkedManifest(lmId string) (provisionedResources, bool)

func (BatchCloudProvider) TearDownResources 

func (b BatchCloudProvider) TearDownResources(job Job) error

type CloudProvider 

type CloudProvider interface {
	//initialize it with some sort of configuration?
	ProvisionResources(jobManager JobManager) error
	Dependencies(lm LinkedModelManifest, eventIndex int, dag DirectedAcyclicGraph) ([]*string, error)
	ResourcesForLinkedManifest(linkedManifestId string) (provisionedResources, bool)
	TearDownResources(job Job) error
	ProcessTask(job *Job, eventIndex int, payloadPath string, linkedManifest LinkedModelManifest) error
}

func InitalizeSession 

func InitalizeSession(config Config) (CloudProvider, error)

type ComputeResourceRequirements 

type ComputeResourceRequirements struct {
	LinkedManifestID   string `json:"linked_manifest_id" yaml:"linked_manifest_id"`
	ComputeEnvironment string `json:"compute_environment" yaml:"compute_environment"` //is this provided as JSON?
	JobDefinition      string `json:"job_definition" yaml:"job_definition"`
	Queue              string `json:"job_queue" yaml:"job_queue"`
}

type Config 

type Config struct {
	CloudProvider Provider    `json:"cloud_provider_type"`
	AwsConfigs    []AwsConfig `json:"aws_configs"`
}

func InitConfig 

func InitConfig(path string) (Config, error)

func (Config) PrimaryConfig 

func (c Config) PrimaryConfig() (AwsConfig, error)

type DirectedAcyclicGraph 

type DirectedAcyclicGraph struct {
	Models          []plugin.ModelIdentifier `json:"models" yaml:"models"`
	LinkedManifests []LinkedModelManifest    `json:"linked_manifests" yaml:"linked_manifests"`
}

func (DirectedAcyclicGraph) GeneratePayload 

func (dag DirectedAcyclicGraph) GeneratePayload(lm LinkedModelManifest, eventIndex int, outputDestination plugin.ResourceInfo) (plugin.ModelPayload, error)

func (DirectedAcyclicGraph) TopologicallySort 

func (dag DirectedAcyclicGraph) TopologicallySort() ([]LinkedModelManifest, error)

type Job 

type Job struct {
	Id                string               `json:"job_identifier" yaml:"job_identifier"`
	EventStartIndex   int                  `json:"event_start_index" yaml:"event_start_index"`
	EventEndIndex     int                  `json:"event_end_index" yaml:"event_end_index"`
	Dag               DirectedAcyclicGraph `json:"directed_acyclic_graph" yaml:"directed_acyclic_graph"`
	OutputDestination plugin.ResourceInfo  `json:"output_destination" yaml:"output_destination"`
}

Job

func (Job) ComputeEvent 

func (job Job) ComputeEvent(eventIndex int, cloud CloudProvider) error

ComputeEvent

func (Job) GeneratePayloads 

func (job Job) GeneratePayloads() error

GeneratePayloads

func (Job) ValidateLinkages 

func (job Job) ValidateLinkages() error

type JobManager 

type JobManager struct {
	// contains filtered or unexported fields
}

JobManager

func Init 

func Init(jobManifest JobManifest, config Config) (JobManager, error)

func (JobManager) LinkedManifestComputeResources 

func (jm JobManager) LinkedManifestComputeResources(linkedManifestId string) (ComputeResourceRequirements, error)

func (JobManager) ProcessJob 

func (jm JobManager) ProcessJob() error

func (JobManager) Validate 

func (jm JobManager) Validate() error

type JobManifest 

type JobManifest struct {
	Id                      string                        `json:"job_identifier" yaml:"job_identifier"`
	EventStartIndex         int                           `json:"event_start_index" yaml:"event_start_index"`
	EventEndIndex           int                           `json:"event_end_index" yaml:"event_end_index"`
	Models                  []plugin.ModelIdentifier      `json:"models" yaml:"models"`
	LinkedManifestResources []plugin.ResourceInfo         `json:"linked_manifests" yaml:"linked_manifests"`
	ComputeResources        []ComputeResourceRequirements `json:"resource_requirements" yaml:"resource_requirements"`
	OutputDestination       plugin.ResourceInfo           `json:"output_destination" yaml:"output_destination"`
}

JobManifest

func (JobManifest) ConvertToJob 

func (jm JobManifest) ConvertToJob() (Job, error)

type LinkedFileData 

type LinkedFileData struct {
	// Id is an internal element generated to identify any data element.
	Id string `json:"id,omitempty" yaml:"id,omitempty"`
	// FileName describes the name of the file that needs to be input or output.
	FileName string `json:"filename" yaml:"filename"`
	// Provider a provider is a specific output data element from a manifest.
	SourceDataId  string                   `json:"source_data_identifier" yaml:"source_data_identifier"`
	InternalPaths []LinkedInternalPathData `json:"internal_paths,omitempty" yaml:"internal_paths,omitempty"`
}

LinkedFileData

func (LinkedFileData) HasInternalPaths 

func (lf LinkedFileData) HasInternalPaths() bool

type LinkedInternalPathData 

type LinkedInternalPathData struct {
	// Id is an internal element generated to identify any data element.
	Id string `json:"id,omitempty" yaml:"id,omitempty"`
	// PathName describes the internal path location to the data needed or produced.
	PathName     string `json:"pathname" yaml:"pathname"`
	SourcePathID string `json:"source_path_identifier,omitempty" yaml:"source_path_identifier,omitempty"`
	SourceFileID string `json:"source_file_identifier" yaml:"source_file_identifier"`
}

LinkedInternalPathData

type LinkedModelManifest 

type LinkedModelManifest struct {
	ManifestID    string `json:"linked_manifest_id" yaml:"linked_manifest_id"`
	plugin.Plugin `json:"plugin" yaml:"plugin"`
	plugin.Model  `json:"model" yaml:"model"`
	Inputs        []LinkedFileData  `json:"inputs" yaml:"inputs"`
	Outputs       []plugin.FileData `json:"outputs" yaml:"outputs"`
}

LinkedModelManifest represents a model manifest that has been linked (i.e. has dependencies)

type MockProvider 

type MockProvider struct {
	Resources map[string]provisionedResources
	// contains filtered or unexported fields
}

func (MockProvider) Dependencies 

func (m MockProvider) Dependencies(lm LinkedModelManifest, eventIndex int, dag DirectedAcyclicGraph) ([]*string, error)

func (MockProvider) ProcessTask 

func (m MockProvider) ProcessTask(job *Job, eventIndex int, payloadPath string, linkedManifest LinkedModelManifest) error

func (MockProvider) ProvisionResources 

func (m MockProvider) ProvisionResources(jobManager JobManager) error

func (MockProvider) ResourcesForLinkedManifest 

func (m MockProvider) ResourcesForLinkedManifest(lmId string) (provisionedResources, bool)

func (MockProvider) TearDownResources 

func (m MockProvider) TearDownResources(job Job) error

type PayloadProcessor 

type PayloadProcessor func(payload plugin.ModelPayload, job Job, eventIndex int, modelManifest LinkedModelManifest) error

type Provider 

type Provider string
const (
	BATCH Provider = "AWS Batch"
	MOCK  Provider = "Mock"
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.