service

package
v0.0.0-...-20989ac Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 15, 2022 License: Apache-2.0 Imports: 45 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// PyFuncEnsemblerServiceEndpoint URL path for the endpoint, e.g "/"
	PyFuncEnsemblerServiceEndpoint string = "/ensemble"
	// PyFuncEnsemblerServicePort Port number the container listens to for requests
	PyFuncEnsemblerServicePort int = 8083
)
View Source 
const (
	// SparkHomeFolder is the home folder of the spark user in the Docker container
	// used in engines/batch-ensembler/Dockerfile
	SparkHomeFolder = "/home/spark"
)

Variables

View Source 
var (
	// EnsemblerFolder is the folder created by the Turing SDK that contains
	// the ensembler dependencies and pickled Python files.
	EnsemblerFolder = "ensembler"
)

Functions

func PaginationScope 

func PaginationScope(options PaginationOptions) func(db *gorm.DB) *gorm.DB

Types

type AlertService 

type AlertService interface {
	Save(alert models.Alert, authorEmail string, dashboardURL string) (*models.Alert, error)
	List(service string) ([]*models.Alert, error)
	FindByID(id models.ID) (*models.Alert, error)
	Update(alert models.Alert, authorEmail string, dashboardURL string) error
	Delete(alert models.Alert, authorEmail string, dashboardURL string) error
	// GetDashboardURL returns the dashboard URL for the router alert.
	//
	// If routerVersion is nil, the dashboard URL should return the dashboard showing metrics
	// for the router across all versions. Else, the dashboard should show metrics for a specific
	// router version.
	GetDashboardURL(
		alert *models.Alert,
		project *mlp.Project,
		environment *merlin.Environment,
		router *models.Router,
		routerVersion *models.RouterVersion) (string, error)
}

func NewGitlabOpsAlertService 

func NewGitlabOpsAlertService(db *gorm.DB, config config.AlertConfig) (AlertService, error)

NewGitlabOpsAlertService Creates a new AlertService that can be used with GitOps based on GitLab. It is assumed that the continuous integration (CI) jobs configured in GitLab can process the committed alert files to register the alerts to the corresponding external alert manager. This CI process is out of scope of Turing.

For every alert object created, a yaml file will be created at the following location: <gitlab_alert_repo_root>/<gitlabPathPrefix>/<environment>/<team>/<service>/<metric>.yaml

Every alert created will be persisted in the database and the configured alert repository in GitLab. In most cases, the alerts persisted in the database will be in sync with the alert files in GitLab (as long as the Git files are not manually modified i.e. the alert files are only updated by calling this service).

type CryptoService 

type CryptoService interface {
	// Encrypt takes an input plaintext string and returs the cipher text or an error
	Encrypt(plaintext string) (string, error)
	// Decrypt takes an input cipher string and returs the plaintext or an error
	Decrypt(ciphertext string) (string, error)
}

CryptoService is used for encrypting / decrypting sensitive data

func NewCryptoService 

func NewCryptoService(encryptionKey string) CryptoService

NewCryptoService creates a new cryptoService using the given encryption key

type DashboardURLValue 

type DashboardURLValue struct {
	Environment string // environment name where the router is deployed
	Cluster     string // Kubernetes cluster name where the router name is deployed
	Project     string // MLP project name where the router is deployed
	Router      string // router name for the alert
	Version     string // router version number
}

DashboardURLValue dashboardURLValue will be passed in as argument to execute dashboardURLTemplate.

type DeploymentService 

type DeploymentService interface {
	DeployRouterVersion(
		project *mlp.Project,
		environment *merlin.Environment,
		routerVersion *models.RouterVersion,
		routerServiceAccountKey string,
		enricherServiceAccountKey string,
		ensemblerServiceAccountKey string,
		pyfuncEnsembler *models.PyFuncEnsembler,
		experimentConfig json.RawMessage,
		eventsCh *EventChannel,
	) (string, error)
	UndeployRouterVersion(
		project *mlp.Project,
		environment *merlin.Environment,
		routerVersion *models.RouterVersion,
		eventsCh *EventChannel,
		isCleanUp bool,
	) error
	DeleteRouterEndpoint(project *mlp.Project,
		environment *merlin.Environment,
		routerVersion *models.RouterVersion,
	) error
}

DeploymentService handles the deployment of the Turing routers and the related components.

func NewDeploymentService 

func NewDeploymentService(
	cfg *config.Config,
	clusterControllers map[string]cluster.Controller,
	ensemblerServiceImageBuilder imagebuilder.ImageBuilder,
) DeploymentService

NewDeploymentService initialises a new endpoints service

type EnsemblersFindByIDOptions 

type EnsemblersFindByIDOptions struct {
	ProjectID *models.ID
}

type EnsemblersListOptions 

type EnsemblersListOptions struct {
	PaginationOptions
	ProjectID     *models.ID            `schema:"project_id" validate:"required"`
	Search        *string               `schema:"search"`
	EnsemblerType *models.EnsemblerType `schema:"type" validate:"omitempty,oneof=pyfunc docker"`
}

EnsemblersListOptions holds query parameters for EnsemblersService.List method

type EnsemblersService 

type EnsemblersService interface {
	// FindByID Find an ensembler matching the given id and options
	FindByID(id models.ID, options EnsemblersFindByIDOptions) (models.EnsemblerLike, error)
	// List ensemblers
	List(options EnsemblersListOptions) (*PaginatedResults, error)
	// Save the given router to the db. Updates the existing record if already exists
	Save(ensembler models.EnsemblerLike) (models.EnsemblerLike, error)
}

EnsemblersService is the data access object for the Ensemblers from the db.

func NewEnsemblersService 

func NewEnsemblersService(db *gorm.DB) EnsemblersService

NewEnsemblersService creates a new ensemblers service

type EnsemblingJobFindByIDOptions 

type EnsemblingJobFindByIDOptions struct {
	ProjectID *models.ID
}

EnsemblingJobFindByIDOptions contains the options allowed when finding ensembling jobs.

type EnsemblingJobListOptions 

type EnsemblingJobListOptions struct {
	PaginationOptions
	ProjectID          *models.ID      `schema:"project_id" validate:"required"`
	EnsemblerID        *models.ID      `schema:"ensembler_id"`
	Statuses           []models.Status `schema:"status"`
	Search             *string         `schema:"search"`
	RetryCountLessThan *int            `schema:"-"`
	UpdatedAtBefore    *time.Time      `schema:"-"`
}

EnsemblingJobListOptions holds query parameters for EnsemblersService.List method.

type EnsemblingJobService 

type EnsemblingJobService interface {
	Save(ensemblingJob *models.EnsemblingJob) error
	Delete(ensemblingJob *models.EnsemblingJob) error
	FindByID(
		id models.ID,
		options EnsemblingJobFindByIDOptions,
	) (*models.EnsemblingJob, error)
	List(options EnsemblingJobListOptions) (*PaginatedResults, error)
	CreateEnsemblingJob(
		job *models.EnsemblingJob,
		projectID models.ID,
		ensembler *models.PyFuncEnsembler,
	) (*models.EnsemblingJob, error)
	MarkEnsemblingJobForTermination(ensemblingJob *models.EnsemblingJob) error
	GetNamespaceByComponent(componentType string, project *mlp.Project) string
	GetDefaultEnvironment() string
	CreatePodLabelSelector(ensemblerName, componentType string) []LabelSelector
	FormatLoggingURL(ensemblerName string, namespace string, componentType string) (string, error)
}

EnsemblingJobService is the data access object for the EnsemblingJob from the db.

func NewEnsemblingJobService 

func NewEnsemblingJobService(
	db *gorm.DB,
	defaultEnvironment string,
	imageBuilderNamespace string,
	loggingURLFormat *string,
	dashboardURLFormat *string,
	defaultConfig config.DefaultEnsemblingJobConfigurations,
	mlpService MLPService,
) EnsemblingJobService

NewEnsemblingJobService creates a new ensembling job service

type EnsemblingMonitoringVariables 

type EnsemblingMonitoringVariables struct {
	// Project is the MLP Project associated with the batch ensembler
	Project string
	// Job is the name of the ensembling job.
	Job string
}

EnsemblingMonitoringVariables the values supplied to BatchEnsemblingConfig.MonitoringURLTemplate

type EventChannel 

type EventChannel struct {
	// contains filtered or unexported fields
}

func NewEventChannel 

func NewEventChannel() *EventChannel

func (*EventChannel) Close 

func (ch *EventChannel) Close()

Close closes the channel in a thread-safe way, and updates isClosed to true.

func (*EventChannel) Read 

func (ch *EventChannel) Read() (*models.Event, bool)

Read reads an event from the stream. If the channel is closed, returns a boolean indicating that the channel is done.

func (*EventChannel) Write 

func (ch *EventChannel) Write(event *models.Event)

Write safely writes to the channel. If the channel is closed, the event will be dropped.

type EventService 

type EventService interface {
	ListEvents(routerID int) ([]*models.Event, error)
	ClearEvents(routerID int) error
	Save(event *models.Event) error
}

func NewEventService 

func NewEventService(db *gorm.DB) EventService

NewEventService creates a new events service

type Experiment 

type Experiment struct {
	ID         string   `json:"id"`
	Name       string   `json:"name"`
	ClientName string   `json:"client_name"`
	UnitType   string   `json:"unit_type"`
	Treatments []string `json:"treatments"` // List of treatment names (i.e. variations) in the experiment.
}

Experiment represents an experiment in Turing. The experiment info can come from different experiment engines.

type ExperimentsService 

type ExperimentsService interface {
	// IsStandardExperimentManager checks if the experiment manager is of the standard type
	IsStandardExperimentManager(engine string) bool
	// IsClientSelectionEnabled checks if the experiment manager is of the standard type and
	// has clients
	IsClientSelectionEnabled(engine string) (bool, error)
	// ListEngines returns a list of the experiment engines available
	ListEngines() []manager.Engine
	// ListClients returns a list of the clients registered on the given experiment engine
	ListClients(engine string) ([]manager.Client, error)
	// ListExperiments returns a list of the experiments registered on the given experiment engine,
	// and for the given clientID if supplied
	ListExperiments(engine string, clientID string) ([]manager.Experiment, error)
	// ListVariables returns a list of the variables registered on the given experiment engine,
	// for the given clientID and/or experiments
	ListVariables(engine string, clientID string, experimentIDs []string) (manager.Variables, error)
	// ValidateExperimentConfig validates the given experiment config for completeness
	ValidateExperimentConfig(engine string, cfg json.RawMessage) error
	// GetExperimentRunnerConfig converts the given experiment config compatible with the Experiment Manager
	// into the format compatible with the ExperimentRunner
	GetExperimentRunnerConfig(engine string, cfg json.RawMessage) (json.RawMessage, error)
}

ExperimentsService provides functionality to work with experiment engines supported by Turing

func NewExperimentsService 

func NewExperimentsService(managerConfig map[string]config.EngineConfig) (ExperimentsService, error)

NewExperimentsService creates a new experiment service from managerConfig. managerConfig is a map of experiment manager name to the JSON string configuration.

type LabelSelector 

type LabelSelector struct {
	Key   string
	Value string
}

LabelSelector refers to the label

type MLPService 

type MLPService interface {
	// GetEnvironments gets all available environments from Merlin
	GetEnvironments() ([]merlin.Environment, error)
	// GetEnvironment gets the environment matching the provided name.
	GetEnvironment(name string) (*merlin.Environment, error)
	// GetProjects list available projects, optionally filtered by given project `name`
	GetProjects(name string) ([]mlp.Project, error)
	// GetProject gets the project matching the provided id.
	GetProject(id models.ID) (*mlp.Project, error)
	// GetSecret gets a secret by project and name.
	GetSecret(projectID models.ID, name string) (string, error)
}

MLPService provides a set of methods to interact with the MLP / Merlin APIs

func NewMLPService 

func NewMLPService(
	mlpBasePath string,
	mlpEncryptionKey string,
	merlinBasePath string,
) (MLPService, error)

NewMLPService returns a service that retrieves information that is shared across MLP projects from (currently) the Merlin API.

type Metric 

type Metric string

type PaginatedResults 

type PaginatedResults struct {
	Results interface{} `json:"results"`
	Paging  Paging      `json:"paging"`
}

type PaginationOptions 

type PaginationOptions struct {
	Page     *int `schema:"page" validate:"omitempty,min=1"`
	PageSize *int `schema:"page_size" validate:"omitempty,min=1"`
}

type Paging 

type Paging struct {
	Total int `json:"total"`
	Page  int `json:"page"`
	Pages int `json:"pages"`
}

type PodLog 

type PodLog struct {
	// Log timestamp in RFC3339 format
	Timestamp time.Time `json:"timestamp"`
	// Environment name of the router running the container that produces this log
	Environment string `json:"environment"`
	// Kubernetes namespace where the pod running the container is created
	Namespace string `json:"namespace"`
	// Pod name running the container that produces this log
	PodName string `json:"pod_name"`
	// Container name that produces this log
	ContainerName string `json:"container_name,omitempty"`
	// Log in text format, either TextPayload or JSONPayload will be set but not both
	TextPayload string `json:"text_payload,omitempty"`
	// Log in JSON format, either TextPayload or JSONPayload will be set but not both
	JSONPayload map[string]interface{} `json:"json_payload,omitempty"`
}

PodLog represents a single log line from a container running in Kubernetes. If the log message is in JSON format, JSONPayload must be populated with the structured JSON log message. Else, TextPayload must be populated with log text. This is the legacy format, use PodLogs for newer features instead.

type PodLogRequest 

type PodLogRequest struct {
	// Kubernetes Namespace, usually the same as the project name
	Namespace string
	// Picks the logs from a selected container in a pod.
	DefaultContainer string
	// Environment that the pod is in, used for the cluster controller selection
	Environment string
	// Labels for Kubernetes pods
	LabelSelectors []LabelSelector
	// This is the template used for persistent logs that are outside Kubernetes
	LoggingURLTemplate *string
	// Container to get the logs from, default to 'user-container', the default container name in Knative
	Container string
	// If true, return the logs from previous terminated container in all pods
	Previous bool
	// (Optional) Timestamp from which to retrieve the logs from, useful for filtering recent logs. The logs retrieved
	// will have timestamp after (but not including) SinceTime.
	SinceTime *time.Time
	// (Optional) Number of lines from the end of the logs to retrieve. Should not be used together with HeadLines.
	// If both TailLines and Headlines are provided, TailLines will be ignored.
	TailLines *int64
	// (Optional) Number of lines from the start of the logs to retrieve.  Should not be used together with TailLines.
	// If both TailLines and Headlines are provided, TailLines will be ignored.
	HeadLines *int64
}

PodLogRequest is the request for logs for a particular set of pods.

type PodLogService 

type PodLogService interface {
	ListPodLogs(request PodLogRequest) ([]*PodLog, error)
}

PodLogService is an interface to retrieve logs from Kubernetes Pods

func NewPodLogService 

func NewPodLogService(clusterControllers map[string]cluster.Controller) PodLogService

NewPodLogService creates a new PodLogService that deals with kubernetes pod logs

type PodLogV2 

type PodLogV2 struct {
	// Log timestamp in RFC3339 format
	Timestamp time.Time `json:"timestamp"`
	// Pod name running the container that produces this log
	PodName string `json:"pod_name"`
	// Log in text format, either TextPayload or JSONPayload will be set but not both
	TextPayload string `json:"text_payload,omitempty"`
}

PodLogV2 represents a single log line from a container running in Kubernetes.

type PodLogsV2 

type PodLogsV2 struct {
	// Environment name of the router running the container that produces this log
	Environment string `json:"environment"`
	// Kubernetes namespace where the pod running the container is created
	Namespace string `json:"namespace"`
	// URL to dashboard since pods might be deleted after use
	// Since there are multiple pods, we will add the unique URLs here
	LoggingURL string `json:"logging_url"`
	// Logs is an array of logs, equivalent to one line of log
	Logs []*PodLogV2 `json:"logs"`
}

PodLogsV2 contains a list of logs in a kubernetes pod along with some extra information. This is the new format for pod logs

func ConvertPodLogsToV2 

func ConvertPodLogsToV2(namespace string, environment string, loggingURL string, podLogs []*PodLog) *PodLogsV2

ConvertPodLogsToV2 converts to the new pod log format

type RouterMonitoringService 

type RouterMonitoringService interface {
	GenerateMonitoringURL(
		projectID models.ID,
		environmentName string,
		routerName string,
		routerVersion *uint,
	) (string, error)
}

func NewRouterMonitoringService 

func NewRouterMonitoringService(
	mlpService MLPService,
	monitoringURLTemplate *template.Template) RouterMonitoringService

type RouterVersionsService 

type RouterVersionsService interface {
	// ListRouterVersions List all RouterVersions associated with the given routerID
	ListRouterVersions(routerID models.ID) ([]*models.RouterVersion, error)
	// ListRouterVersionsWithStatus Lists the RouterVersions for the given Router matching the given status.
	ListRouterVersionsWithStatus(routerID models.ID, status models.RouterVersionStatus) ([]*models.RouterVersion, error)
	// Save the given RouterVersion to the db. Updates the existing record if already exists.
	Save(routerVersion *models.RouterVersion) (*models.RouterVersion, error)
	// FindByID Finds the RouterVersion matching the given id.
	FindByID(routerVersionID models.ID) (*models.RouterVersion, error)
	// FindByRouterIDAndVersion Finds the RouterVersion for the given Router matching the given version.
	FindByRouterIDAndVersion(routerID models.ID, version uint) (*models.RouterVersion, error)
	// FindLatestVersionByRouterID Finds the latest RouterVersion for the given Router matching the given version.
	FindLatestVersionByRouterID(routerID models.ID) (*models.RouterVersion, error)
	// Delete Deletes the given RouterVersion from the db. This method deletes all child objects (enricher, ensembler).
	Delete(routerVersion *models.RouterVersion) error
}

RouterVersionsService is the data access object for RouterVersions from the db.

func NewRouterVersionsService 

func NewRouterVersionsService(
	db *gorm.DB,
	mlpService MLPService,
	monitoringURLFormat *string,
) RouterVersionsService

type RoutersService 

type RoutersService interface {
	//ListRouters List routers within the given project and environment.
	ListRouters(projectID models.ID, environmentName string) ([]*models.Router, error)
	// Save the given router to the db. Updates the existing record if already exists.
	Save(router *models.Router) (*models.Router, error)
	//FindByID Find a router matching the given router id.
	FindByID(routerID models.ID) (*models.Router, error)
	//FindByProjectAndName Find a router within the given project that matches the given name.
	FindByProjectAndName(projectID models.ID, routerName string) (*models.Router, error)
	//FindByProjectAndEnvironmentAndName Find a router within the given project and environment
	// that matches the given name.
	FindByProjectAndEnvironmentAndName(
		projectID models.ID,
		environmentName string,
		routerName string,
	) (*models.Router, error)
	// Delete a router. This deletes all child objects of the router (router versions, ensemblers and enrichers)
	// (Transactional).
	Delete(router *models.Router) error
}

RoutersService is the data access object for the Routers from the db.

func NewRoutersService 

func NewRoutersService(db *gorm.DB, mlpService MLPService, monitoringURLFormat *string) RoutersService

NewRoutersService creates a new router service

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.