workers

Documentation

Index

• type Context
  • func (context Context) Clone() (clonedContext Context)
  • func (context Context) DatasetDownloadListener()
  • func (context Context) DatasetDownloadWorker(dataset types.Dataset)
  • func (context Context) DatasetGitWorker(dataset types.Dataset)
  • func (context Context) DatasetLocalCopyWorker(dataset types.Dataset)
  • func (context Context) DatasetUnpackListener()
  • func (context Context) DatasetUnpackWorker(dataset types.Dataset)
  • func (context Context) DatasetValidatorListener()
  • func (context Context) DatasetValidatorkWorker(dataset types.Dataset)
  • func (context Context) JobPausingWorker(job types.Job)
  • func (context Context) JobResumingWorker(job types.Job)
  • func (context Context) JobStatusMaintainerListener()
  • func (context Context) JobTerminatingWorker(job types.Job)
  • func (context Context) ModuleDownloadListener()
  • func (context Context) ModuleDownloadWorker(module types.Module)
  • func (context Context) ModuleLocalCopyWorker(module types.Module)
  • func (context Context) ModuleRegistryPullWorker(module types.Module)
  • func (context Context) ModuleValidateListener()
  • func (context Context) ModuleValidateWorker(module types.Module)
  • func (context Context) OptimizerRunListener(optimizerID string)
  • func (context Context) OptimizerRunWorker(optimizerID string, numProcesses, numTasks int)
  • func (context Context) ProcessKeepaliveWorker(period time.Duration)
  • func (context Context) TaskPausingWorker(task types.Task)
  • func (context Context) TaskRunListener()
  • func (context Context) TaskRunWorker(task types.Task)
  • func (context Context) TaskStatusMaintainerListener()
  • func (context Context) TaskTerminatingWorker(task types.Task)
  • func (context Context) TerminateDeadProcessesWorker(period time.Duration)

Types

type Context

type Context struct {
	ModelContext   model.Context
	StorageContext storage.Context
	ProcessID      bson.ObjectId
	Period         time.Duration
	Logger         logger.Logger
	GpuDevices     []string
}

Context contains all information needed to do work.

func (Context) Clone

func (context Context) Clone() (clonedContext Context)

Clone makes a copy of the mongo session.

func (Context) DatasetDownloadListener

func (context Context) DatasetDownloadListener()

DatasetDownloadListener periodically checks if there are any datasets which have been created with source set to "download" but the download hasn't been successfully performed yet.

func (Context) DatasetDownloadWorker

func (context Context) DatasetDownloadWorker(dataset types.Dataset)

DatasetDownloadWorker performs the actual dataset download.

func (Context) DatasetGitWorker

func (context Context) DatasetGitWorker(dataset types.Dataset)

DatasetGitWorker performs the actual dataset fetch from a git lfs repository.

func (Context) DatasetLocalCopyWorker

func (context Context) DatasetLocalCopyWorker(dataset types.Dataset)

DatasetLocalCopyWorker copies the local dataset if it is a directory.

func (Context) DatasetUnpackListener

func (context Context) DatasetUnpackListener()

DatasetUnpackListener periodically checks if there are any datasets which have been transferred (either through upload or download) and unpacks them.

func (Context) DatasetUnpackWorker

func (context Context) DatasetUnpackWorker(dataset types.Dataset)

DatasetUnpackWorker performs the unpacking operation.

func (Context) DatasetValidatorListener

func (context Context) DatasetValidatorListener()

DatasetValidatorListener periodically checks if there are any datasets which have been unpacked in order to validate them.

func (Context) DatasetValidatorkWorker

func (context Context) DatasetValidatorkWorker(dataset types.Dataset)

DatasetValidatorkWorker performs the unpacking operation.

func (Context) JobPausingWorker

func (context Context) JobPausingWorker(job types.Job)

JobPausingWorker handles all pausing jobs by pausing their tasks.

func (Context) JobResumingWorker

func (context Context) JobResumingWorker(job types.Job)

JobResumingWorker handles all resuming jobs by pausing their tasks.

func (Context) JobStatusMaintainerListener

func (context Context) JobStatusMaintainerListener()

JobStatusMaintainerListener periodically checks if there are any jobs whose status has changed to pausing, resuming or terminating and handles them.

func (Context) JobTerminatingWorker

func (context Context) JobTerminatingWorker(job types.Job)

JobTerminatingWorker handles all terminating jobs by pausing their tasks.

func (Context) ModuleDownloadListener

func (context Context) ModuleDownloadListener()

ModuleDownloadListener periodically checks if there are any modules which have been created with source set to "download" but the download hasn't been successfully performed yet.

func (Context) ModuleDownloadWorker

func (context Context) ModuleDownloadWorker(module types.Module)

ModuleDownloadWorker performs the actual module download.

func (Context) ModuleLocalCopyWorker

func (context Context) ModuleLocalCopyWorker(module types.Module)

ModuleLocalCopyWorker copies the local module if it is a directory.

func (Context) ModuleRegistryPullWorker

func (context Context) ModuleRegistryPullWorker(module types.Module)

ModuleRegistryPullWorker copies the local module if it is a directory.

func (Context) ModuleValidateListener

func (context Context) ModuleValidateListener()

ModuleValidateListener periodically checks if there are any modules which have been transferred but have not yet been validated. It performs various checks to make sure the model is ready to become activated.

func (Context) ModuleValidateWorker

func (context Context) ModuleValidateWorker(module types.Module)

ModuleValidateWorker performs the actual module validation. It makes sure the model has a defined config space and input and output schema.

func (Context) OptimizerRunListener

func (context Context) OptimizerRunListener(optimizerID string)

OptimizerRunListener periodically checks if there are any modules which have been transferred but have not yet been validated. It performs various checks to make sure the model is ready to become activated.

func (Context) OptimizerRunWorker

func (context Context) OptimizerRunWorker(optimizerID string, numProcesses, numTasks int)

OptimizerRunWorker runs the optimization sequence.

func (Context) ProcessKeepaliveWorker

func (context Context) ProcessKeepaliveWorker(period time.Duration)

ProcessKeepaliveWorker updates the keepalive-time field of the process thus notifying the system that it is still running.

func (Context) TaskPausingWorker

func (context Context) TaskPausingWorker(task types.Task)

TaskPausingWorker handles all pausing tasks by pausing their tasks.

func (Context) TaskRunListener

func (context Context) TaskRunListener()

TaskRunListener periodically checks if there are any tasks which are in the "scheduled" state which means they are ready to run.

func (Context) TaskRunWorker

func (context Context) TaskRunWorker(task types.Task)

TaskRunWorker takes a task and runs it through all the stages.

func (Context) TaskStatusMaintainerListener

func (context Context) TaskStatusMaintainerListener()

TaskStatusMaintainerListener periodically checks if there are any tasks whose status has changed to pausing or terminating and handles them.

func (Context) TaskTerminatingWorker

func (context Context) TaskTerminatingWorker(task types.Task)

TaskTarminatingWorker handles all terminating tasks.

func (Context) TerminateDeadProcessesWorker

func (context Context) TerminateDeadProcessesWorker(period time.Duration)

TerminateDeadProcessesWorker goes through all processes that have stopped making keepalive updates and sets their status to terminated.