job

Documentation

Index

• func Cancel(jobs Store) http.Handler
• func CreateJob(jobs Store, projects project.Store) http.Handler
• func Delete(jobs Store) http.Handler
• func Get(jobs Store, sm *sagemaker.Client) http.Handler
• func GetAll(jobs Store) http.Handler
• func GetLog(logfileDir string) http.Handler
• func Put(jobs Store) http.Handler
• func SubmitJob(jobs Store, projects project.Store, notifier pubsub.Publisher, ...) http.Handler
• type Commit
• type GitHub
• type Job
• type Metric
• type Postgres
  • func (store *Postgres) CancelJobs(accountID, projectID int64, jobIDs []int64) error
  • func (store *Postgres) CreateJob(accountID, projectID int64, submitter, deliveryID, revisionID string) (jobID int64, err error)
  • func (store *Postgres) DeleteJobs(accountID, projectID int64, jobIDs []int64) error
  • func (store *Postgres) Get(jobID int64) (Job, error)
  • func (store *Postgres) GetAll(accountID, projectID int64) ([]Job, error)
  • func (store *Postgres) UpdateJob(job event.JobUpdated) error
• type Project
• type ResourceConfig
• type SageMakerTrainingJob
• type SageMakerTransformJob
• type SageMakerTuningJob
• type Sender
• type Store
• type WebhookValidator

Functions

func Cancel

func Cancel(jobs Store) http.Handler

Cancel jobs

func CreateJob

func CreateJob(jobs Store, projects project.Store) http.Handler

CreateJob creates a job (not to be confused with submitting a job which happens earlier e.g. from GitHub). For CreateJob to be called, a job submission has been authorized and submitted to the scheduler service which decides when to actually create a job entry in the database (this call).

func Delete

func Delete(jobs Store) http.Handler

Delete jobs

func Get

func Get(jobs Store, sm *sagemaker.Client) http.Handler

Get returns a job's info.

func GetAll

func GetAll(jobs Store) http.Handler

GetAll returns a project's jobs. TODO(ilazakis): paging, hard limit or both; job count can get out of hand quickly.

func GetLog

func GetLog(logfileDir string) http.Handler

GetLog returns a job's log file

func Put

func Put(jobs Store) http.Handler

Put updates the job's details based on `event.JobUpdated`

func SubmitJob

func SubmitJob(jobs Store, projects project.Store, notifier pubsub.Publisher, webhookValidator WebhookValidator, webhookSecret string) http.Handler

SubmitJob submits a job on the 'arrivals' queue for the scheduler service to pick up.

Types

type Commit

type Commit struct {
	ID string `json:"id"`
}

type GitHub

type GitHub struct {
	DeliveryID string
	Branch     string     `json:"ref"`
	Sender     Sender     `json:"sender"`
	Repo       repository `json:"repository"`
	Commit     Commit     `json:"head_commit"`
}

GitHub – incoming GitHub webhook payload. Only exporting fields Kenza needs. https://developer.github.com/v3/activity/events/types/#pushevent

type Job

type Job struct {
	ID                    int64                 `json:"id"`
	Status                string                `json:"status"`
	Type                  string                `json:"type"`
	Submitter             string                `json:"submitter"`
	Project               Project               `json:"project"`
	CommitID              string                `json:"commitID"`
	Started               time.Time             `json:"started"`
	Created               string                `json:"created"`
	Updated               string                `json:"updated"`
	SageMakerID           string                `json:"sagemakerID"`
	Endpoint              string                `json:"endpoint,omitempty"`
	Region                string                `json:"region"`
	EndpointInfo          interface{}           `json:"endpoint_info"`
	SageMakerTuningJob    SageMakerTuningJob    `json:"sagemaker_tuning_job,omitempty"`
	SageMakerTrainingJob  SageMakerTrainingJob  `json:"sagemaker_training_job,omitempty"`
	SageMakerTransformJob SageMakerTransformJob `json:"sagemaker_transform_job,omitempty"`
}

The Job model.

type Metric

type Metric struct {
	Name  string  `json:"name"`
	Value float64 `json:"value"`
}

Metric info, maps SageMaker's `MetricData` struct

type Postgres

type Postgres struct {
	DB *sql.DB
}

Postgres is the postgres jobs `Store` implementation.

func (*Postgres) CancelJobs

func (store *Postgres) CancelJobs(accountID, projectID int64, jobIDs []int64) error

CancelJobs delets a list of jobs.

func (*Postgres) CreateJob

func (store *Postgres) CreateJob(accountID, projectID int64, submitter, deliveryID, revisionID string) (jobID int64, err error)

CreateJob creates a new training job.

func (*Postgres) DeleteJobs

func (store *Postgres) DeleteJobs(accountID, projectID int64, jobIDs []int64) error

DeleteJobs delets a list of jobs.

func (*Postgres) Get

func (store *Postgres) Get(jobID int64) (Job, error)

Get returns a job's info.

func (*Postgres) GetAll

func (store *Postgres) GetAll(accountID, projectID int64) ([]Job, error)

GetAll returns a project's jobs.

func (*Postgres) UpdateJob

func (store *Postgres) UpdateJob(job event.JobUpdated) error

UpdateJob persists changes to the passed job.

type Project

type Project struct {
	ID   int64  `json:"id"`
	Name string `json:"name"`
	Repo string `json:"repo"`
}

Project model from a Job's point of view.

type ResourceConfig

type ResourceConfig struct {
	VolumeSizeInGB int64  `json:"volume_size_gb"`
	InstanceType   string `json:"instance_type"`
	InstanceCount  int64  `json:"instance_count"`
}

ResourceConfig describes the resources, including ML compute instances and ML storage volumes, to use for model training.

type SageMakerTrainingJob

type SageMakerTrainingJob struct {
	Metrics                   []Metric           `json:"metrics"`
	HyperParameters           map[string]*string `json:"hyperparameters"`
	S3ModelLocation           string             `json:"s3_model_location"`
	TrainingTimeInSeconds     int64              `json:"training_time_seconds"`
	ResourceConfig            ResourceConfig     `json:"resource_config"`
	DescribeTrainingJobOutput interface{}
}

SageMakerTrainingJob model

type SageMakerTransformJob

type SageMakerTransformJob struct {
	S3InputLocation            string `json:"s3_input_location"`
	S3OutputLocation           string `json:"s3_output_location"`
	DescribeTransformJobOutput interface{}
}

SageMakerTransformJob model

type SageMakerTuningJob

type SageMakerTuningJob struct {
	S3InputLocation                       string `json:"s3_input_location"`
	S3OutputLocation                      string `json:"s3_output_location"`
	DescribeHyperParameterTuningJobOutput interface{}
}

SageMakerTuningJob model

type Sender

type Sender struct {
	Username string `json:"login"`
}

type Store

type Store interface {
	// Delete a list of jobs.
	DeleteJobs(accountID, projectID int64, jobIDs []int64) error

	// Cancel a list of jobs.
	CancelJobs(accountID, projectID int64, jobIDs []int64) error

	// Get a job's info.
	Get(jobID int64) (Job, error)

	// GetAll returns a project's jobs.
	GetAll(accountID, projectID int64) ([]Job, error)

	// Create a job based on the GitHub hook info.
	CreateJob(accountID, projectID int64, submitter, deliveryID, revisionID string) (jobID int64, err error)

	// UpdateJob persists changes to a job.
	UpdateJob(job event.JobUpdated) error
}

Store - job store abstraction

type WebhookValidator

type WebhookValidator func(r *http.Request, secretToken []byte) (payload []byte, err error)

WebhookValidator validates an incoming webhook request. See https://developer.github.com/webhooks/securing/ for details.