server

package
v0.3.45 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 13, 2023 License: Apache-2.0 Imports: 51 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type AggregatedQueueServer 

type AggregatedQueueServer struct {

	// For storing reports of scheduling attempts.
	SchedulingReportsRepository *scheduler.SchedulingReportsRepository[*api.Job]
	// Stores the most recent NodeDb for each executor.
	// Used to check if a job could ever be scheduled at job submit time.
	SubmitChecker *scheduler.SubmitChecker
	// contains filtered or unexported fields
}

func NewAggregatedQueueServer 

func NewAggregatedQueueServer(
	permissions authorization.PermissionChecker,
	schedulingConfig configuration.SchedulingConfig,
	jobRepository repository.JobRepository,
	queueRepository repository.QueueRepository,
	usageRepository repository.UsageRepository,
	eventStore repository.EventStore,
	schedulingInfoRepository repository.SchedulingInfoRepository,
) *AggregatedQueueServer

func (*AggregatedQueueServer) RenewLease 

func (q *AggregatedQueueServer) RenewLease(ctx context.Context, request *api.RenewLeaseRequest) (*api.IdList, error)

func (*AggregatedQueueServer) ReportDone 

func (q *AggregatedQueueServer) ReportDone(ctx context.Context, idList *api.IdList) (*api.IdList, error)

func (*AggregatedQueueServer) ReturnLease 

func (q *AggregatedQueueServer) ReturnLease(ctx context.Context, request *api.ReturnLeaseRequest) (*types.Empty, error)

func (*AggregatedQueueServer) StreamingLeaseJobs added in v0.3.15 

func (q *AggregatedQueueServer) StreamingLeaseJobs(stream api.AggregatedQueue_StreamingLeaseJobsServer) error

StreamingLeaseJobs is called by the executor to request jobs for it to run. It streams jobs to the executor as quickly as it can and then waits to receive ids back. Only jobs for which an id was sent back are marked as leased.

This function should be used instead of the LeaseJobs function in most cases.

type ErrUnauthorized added in v0.3.35 

type ErrUnauthorized struct {
	// Principal that attempted the action
	Principal authorization.Principal
	// Reasons that the principal was not allowed to perform the action
	// For example ["does not own the queue and have SubmitJobs permissions", "does not have SubmitAnyJobs permissions"]
	Reasons []string
}

ErrUnauthorized represents an error that occurs when a client tries to perform some action through the gRPC API for which it does not have permissions. Produces error messages of the form "Tom" does not own the queue and have SubmitJobs permissions, "Tom" does not have SubmitAnyJobs permissions

The caller of a function that may produce this error should capture is using errors.As and prepend whatever action the principal was attempting.

func MergePermissionErrors added in v0.2.19 

func MergePermissionErrors(errs ...*ErrUnauthorized) *ErrUnauthorized

func (*ErrUnauthorized) Error added in v0.3.35 

func (err *ErrUnauthorized) Error() string

type EventServer 

type EventServer struct {
	// contains filtered or unexported fields
}

func NewEventServer 

func NewEventServer(
	permissions authorization.PermissionChecker,
	eventRepository repository.EventRepository,
	eventStore repository.EventStore,
	queueRepository repository.QueueRepository,
	jobRepository repository.JobRepository,
) *EventServer

func (*EventServer) GetJobSetEvents 

func (s *EventServer) GetJobSetEvents(request *api.JobSetRequest, stream api.Event_GetJobSetEventsServer) error

GetJobSetEvents streams back all events associated with a particular job set.

func (*EventServer) Health added in v0.3.35 

func (s *EventServer) Health(ctx context.Context, cont_ *types.Empty) (*api.HealthCheckResponse, error)

func (*EventServer) Report 

func (s *EventServer) Report(ctx context.Context, message *api.EventMessage) (*types.Empty, error)

func (*EventServer) ReportMultiple added in v0.0.11 

func (s *EventServer) ReportMultiple(ctx context.Context, message *api.EventList) (*types.Empty, error)

func (*EventServer) Watch added in v0.2.11 

func (s *EventServer) Watch(req *api.WatchRequest, stream api.Event_WatchServer) error

type EventsPrinter added in v0.3.2 

type EventsPrinter struct {
	Client           pulsar.Client
	Topic            string
	SubscriptionName string
	// Logger from which the loggers used by this service are derived
	// (e.g., using srv.Logger.WithField), or nil, in which case the global logrus logger is used.
	Logger *logrus.Entry
}

EventsPrinter is a service that prints all events passing through pulsar to a logger. This service is only meant for use during development; it will be slow when the number of events is large.

func (*EventsPrinter) Run added in v0.3.2 

func (srv *EventsPrinter) Run(ctx context.Context) error

Run the service that reads from Pulsar and updates Armada until the provided context is cancelled.

type PulsarSubmitServer added in v0.3.0 

type PulsarSubmitServer struct {
	api.UnimplementedSubmitServer
	Producer        pulsar.Producer
	Permissions     authorization.PermissionChecker
	QueueRepository repository.QueueRepository
	// Maximum size of Pulsar messages
	MaxAllowedMessageSize uint
	// Fall back to the legacy submit server for queue administration endpoints.
	SubmitServer *SubmitServer
	// Used for job submission deduplication.
	KVStore *pgkeyvalue.PGKeyValueStore
	// Used to check at job submit time if the job could ever be scheduled.
	// Currently only used for gang jobs.
	SubmitChecker *scheduler.SubmitChecker
}

PulsarSubmitServer is a service that accepts API calls according to the original Armada submit API and publishes messages to Pulsar based on those calls. TODO: Consider returning a list of message ids of the messages generated TODO: Include job set as the message key for each message

func (*PulsarSubmitServer) Authorize added in v0.3.0 

func (srv *PulsarSubmitServer) Authorize(
	ctx context.Context,
	queueName string,
	anyPerm permission.Permission,
	perm queue.PermissionVerb,
) (userId string, groups []string, err error)

Authorize authorises a user request to submit a state transition message to the log. User information used for authorization is extracted from the provided context. Checks that the user has either anyPerm (e.g., permissions.SubmitAnyJobs) or perm (e.g., PermissionVerbSubmit) for this queue. Returns the userId and groups extracted from the context.

func (*PulsarSubmitServer) CancelJobSet added in v0.3.25 

func (srv *PulsarSubmitServer) CancelJobSet(ctx context.Context, req *api.JobSetCancelRequest) (*types.Empty, error)

func (*PulsarSubmitServer) CancelJobs added in v0.3.0 

func (srv *PulsarSubmitServer) CancelJobs(ctx context.Context, req *api.JobCancelRequest) (*api.CancellationResult, error)

func (*PulsarSubmitServer) CreateQueue added in v0.3.0 

func (srv *PulsarSubmitServer) CreateQueue(ctx context.Context, req *api.Queue) (*types.Empty, error)

Fallback methods. Calls into an embedded server.SubmitServer.

func (*PulsarSubmitServer) CreateQueues added in v0.3.15 

func (srv *PulsarSubmitServer) CreateQueues(ctx context.Context, req *api.QueueList) (*api.BatchQueueCreateResponse, error)

func (*PulsarSubmitServer) DeleteQueue added in v0.3.0 

func (srv *PulsarSubmitServer) DeleteQueue(ctx context.Context, req *api.QueueDeleteRequest) (*types.Empty, error)

func (*PulsarSubmitServer) GetQueue added in v0.3.0 

func (srv *PulsarSubmitServer) GetQueue(ctx context.Context, req *api.QueueGetRequest) (*api.Queue, error)

func (*PulsarSubmitServer) GetQueueInfo added in v0.3.0 

func (srv *PulsarSubmitServer) GetQueueInfo(ctx context.Context, req *api.QueueInfoRequest) (*api.QueueInfo, error)

func (*PulsarSubmitServer) ReprioritizeJobs added in v0.3.0 

func (srv *PulsarSubmitServer) ReprioritizeJobs(ctx context.Context, req *api.JobReprioritizeRequest) (*api.JobReprioritizeResponse, error)

func (*PulsarSubmitServer) SubmitApiEvents added in v0.3.8 

func (srv *PulsarSubmitServer) SubmitApiEvents(ctx context.Context, apiEvents []*api.EventMessage) error

SubmitApiEvents converts several api.EventMessage into Pulsar state transition messages and publishes those to Pulsar.

func (*PulsarSubmitServer) SubmitJobs added in v0.3.0 

func (srv *PulsarSubmitServer) SubmitJobs(ctx context.Context, req *api.JobSubmitRequest) (*api.JobSubmitResponse, error)

func (*PulsarSubmitServer) UpdateQueue added in v0.3.0 

func (srv *PulsarSubmitServer) UpdateQueue(ctx context.Context, req *api.Queue) (*types.Empty, error)

func (*PulsarSubmitServer) UpdateQueues added in v0.3.15 

func (srv *PulsarSubmitServer) UpdateQueues(ctx context.Context, req *api.QueueList) (*api.BatchQueueUpdateResponse, error)

type SubmitFromLog added in v0.3.0 

type SubmitFromLog struct {
	SubmitServer *SubmitServer
	Consumer     pulsar.Consumer
	// Logger from which the loggers used by this service are derived
	// (e.g., using srv.Logger.WithField), or nil, in which case the global logrus logger is used.
	Logger *logrus.Entry
}

SubmitFromLog is a service that reads messages from Pulsar and updates the state of the Armada server accordingly (in particular, it writes to Redis). Calls into an embedded Armada submit server object.

func (*SubmitFromLog) BatchedCancelJobsById added in v0.3.25 

func (srv *SubmitFromLog) BatchedCancelJobsById(ctx context.Context, userId string, jobIds []string) (bool, error)

func (*SubmitFromLog) CancelJobSet added in v0.3.0 

func (srv *SubmitFromLog) CancelJobSet(ctx context.Context, userId string, queueName string, jobSetName string) (bool, error)

func (*SubmitFromLog) CancelJobSets added in v0.3.0 

func (srv *SubmitFromLog) CancelJobSets(ctx context.Context, userId string,
	queueName string, jobSetName string, _ []*armadaevents.CancelJobSet,
) (bool, error)

CancelJobSets processes several CancelJobSet events. Because event sequences are specific to queue and job set, all CancelJobSet events in a sequence are equivalent, and we only need to call CancelJobSet once.

func (*SubmitFromLog) CancelJobs added in v0.3.0 

func (srv *SubmitFromLog) CancelJobs(ctx context.Context, userId string, es []*armadaevents.CancelJob) (bool, error)

CancelJobs cancels all jobs specified by the provided events in a single operation.

func (*SubmitFromLog) CancelJobsById added in v0.3.0 

func (srv *SubmitFromLog) CancelJobsById(ctx context.Context, userId string, jobIds []string) ([]string, error)

CancelJobsById cancels all jobs with the specified ids.

func (*SubmitFromLog) ProcessSequence added in v0.3.0 

func (srv *SubmitFromLog) ProcessSequence(ctx context.Context, sequence *armadaevents.EventSequence) bool

ProcessSequence processes all events in a particular sequence. For efficiency, we may process several events at a time. To maintain ordering, we only do so for subsequences of consecutive events of equal type. The returned bool indicates if the corresponding Pulsar message should be ack'd or not.

func (*SubmitFromLog) ProcessSubSequence added in v0.3.0 

func (srv *SubmitFromLog) ProcessSubSequence(ctx context.Context, i int, sequence *armadaevents.EventSequence) (j int, err error)

ProcessSubSequence processes sequence.Events[i:j-1], where j is the index of the first event in the sequence of a type different from that of sequence.Events[i], or len(sequence.Events) if no such event exists in the sequence, and returns j.

Processing one such subsequence at a time preserves ordering between events of different types. For example, SubmitJob events are processed before CancelJob events that occur later in the sequence.

Events are processed by calling into the embedded srv.SubmitServer.

Not all events are handled by this processor since the legacy scheduler writes some transitions directly to the db.

func (*SubmitFromLog) ReprioritizeJobSet added in v0.3.0 

func (srv *SubmitFromLog) ReprioritizeJobSet(
	ctx context.Context,
	userId string,
	queueName string,
	jobSetName string,
	e *armadaevents.ReprioritiseJobSet,
) (bool, error)

func (*SubmitFromLog) ReprioritizeJobSets added in v0.3.0 

func (srv *SubmitFromLog) ReprioritizeJobSets(
	ctx context.Context,
	userId string,
	queueName string,
	jobSetName string,
	es []*armadaevents.ReprioritiseJobSet,
) (bool, error)

ReprioritizeJobSets updates the priority of several job sets. Returns a multierror containing all errors that occurred. Since repeating this operation is safe (setting the priority is idempotent), the bool indicating if events were processed is set to false if any job set failed.

func (*SubmitFromLog) ReprioritizeJobs added in v0.3.0 

func (srv *SubmitFromLog) ReprioritizeJobs(ctx context.Context, userId string, es []*armadaevents.ReprioritiseJob) (bool, error)

ReprioritizeJobs updates the priority of one of more jobs.

func (*SubmitFromLog) Run added in v0.3.0 

func (srv *SubmitFromLog) Run(ctx context.Context) error

Run the service that reads from Pulsar and updates Armada until the provided context is cancelled.

func (*SubmitFromLog) SubmitJobs added in v0.3.0 

func (srv *SubmitFromLog) SubmitJobs(
	ctx context.Context,
	userId string,
	groups []string,
	queueName string,
	jobSetName string,
	es []*armadaevents.SubmitJob,
) (bool, error)

SubmitJobs processes several job submit events in bulk. It returns a boolean indicating if the events were processed and any error that occurred during processing. Specifically, events are not processed if writing to the database results in a network-related error. For any other error, the jobs are marked as failed and the events are considered to have been processed.

func (*SubmitFromLog) UpdateJobStartTimes added in v0.3.44 

func (srv *SubmitFromLog) UpdateJobStartTimes(ctx context.Context, es []*armadaevents.EventSequence_Event) (bool, error)

UpdateJobStartTimes records the start time (in Redis) of one of more jobs.

type SubmitServer 

type SubmitServer struct {
	// contains filtered or unexported fields
}

func NewSubmitServer 

func NewSubmitServer(
	permissions authorization.PermissionChecker,
	jobRepository repository.JobRepository,
	queueRepository repository.QueueRepository,
	eventStore repository.EventStore,
	schedulingInfoRepository repository.SchedulingInfoRepository,
	cancelJobsBatchSize int,
	queueManagementConfig *configuration.QueueManagementConfig,
	schedulingConfig *configuration.SchedulingConfig,
) *SubmitServer

func (*SubmitServer) CancelJobSet added in v0.3.25 

func (server *SubmitServer) CancelJobSet(ctx context.Context, request *api.JobSetCancelRequest) (*types.Empty, error)

func (*SubmitServer) CancelJobs 

func (server *SubmitServer) CancelJobs(ctx context.Context, request *api.JobCancelRequest) (*api.CancellationResult, error)

CancelJobs cancels jobs identified by the request. If the request contains a job ID, only the job with that ID is cancelled. If the request contains a queue name and a job set ID, all jobs matching those are cancelled.

func (*SubmitServer) CreateQueue 

func (server *SubmitServer) CreateQueue(ctx context.Context, request *api.Queue) (*types.Empty, error)

func (*SubmitServer) CreateQueues added in v0.3.15 

func (server *SubmitServer) CreateQueues(ctx context.Context, request *api.QueueList) (*api.BatchQueueCreateResponse, error)

func (*SubmitServer) DeleteQueue added in v0.1.14 

func (server *SubmitServer) DeleteQueue(ctx context.Context, request *api.QueueDeleteRequest) (*types.Empty, error)

func (*SubmitServer) GetQueue added in v0.2.3 

func (server *SubmitServer) GetQueue(ctx context.Context, req *api.QueueGetRequest) (*api.Queue, error)

func (*SubmitServer) GetQueueInfo added in v0.1.2 

func (server *SubmitServer) GetQueueInfo(ctx context.Context, req *api.QueueInfoRequest) (*api.QueueInfo, error)

func (*SubmitServer) Health added in v0.3.35 

func (server *SubmitServer) Health(ctx context.Context, _ *types.Empty) (*api.HealthCheckResponse, error)

func (*SubmitServer) ReprioritizeJobs added in v0.1.39 

func (server *SubmitServer) ReprioritizeJobs(ctx context.Context, request *api.JobReprioritizeRequest) (*api.JobReprioritizeResponse, error)

ReprioritizeJobs updates the priority of one of more jobs. Returns a map from job ID to any error (or nil if the call succeeded).

func (*SubmitServer) SubmitJobs added in v0.0.3 

func (server *SubmitServer) SubmitJobs(ctx context.Context, req *api.JobSubmitRequest) (*api.JobSubmitResponse, error)

func (*SubmitServer) UpdateQueue added in v0.2.3 

func (server *SubmitServer) UpdateQueue(ctx context.Context, request *api.Queue) (*types.Empty, error)

func (*SubmitServer) UpdateQueues added in v0.3.15 

func (server *SubmitServer) UpdateQueues(ctx context.Context, request *api.QueueList) (*api.BatchQueueUpdateResponse, error)

type UsageServer 

type UsageServer struct {
	// contains filtered or unexported fields
}

func NewUsageServer 

func NewUsageServer(
	permissions authorization.PermissionChecker,
	priorityHalfTime time.Duration,
	schedulingConfig *configuration.SchedulingConfig,
	usageRepository repository.UsageRepository,
	queueRepository repository.QueueRepository,
) *UsageServer

func (*UsageServer) ReportUsage 

func (s *UsageServer) ReportUsage(ctx context.Context, report *api.ClusterUsageReport) (*types.Empty, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.