swarming

package
v0.0.0-...-03d6fc4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 23, 2019 License: BSD-3-Clause Imports: 25 Imported by: 0

Documentation

Overview

Code generated by mockery v1.0.0 (and then touched up by hand) 

Utilities for isolating and swarming. See swarming_test.go for usage examples.

Index

Constants

View Source 
const (
	API_BASE_PATH_PATTERN = "https://%s/_ah/api/swarming/v1/"
	AUTH_SCOPE            = "https://www.googleapis.com/auth/userinfo.email"

	DIMENSION_POOL_KEY                 = "pool"
	DIMENSION_POOL_VALUE_SKIA          = "Skia"
	DIMENSION_POOL_VALUE_SKIA_CT       = "SkiaCT"
	DIMENSION_POOL_VALUE_SKIA_INTERNAL = "SkiaInternal"
	DIMENSION_POOL_VALUE_CT            = "CT"

	TASK_STATE_BOT_DIED    = "BOT_DIED"
	TASK_STATE_CANCELED    = "CANCELED"
	TASK_STATE_COMPLETED   = "COMPLETED"
	TASK_STATE_EXPIRED     = "EXPIRED"
	TASK_STATE_KILLED      = "KILLED"
	TASK_STATE_NO_RESOURCE = "NO_RESOURCE"
	TASK_STATE_PENDING     = "PENDING"
	TASK_STATE_RUNNING     = "RUNNING"
	TASK_STATE_TIMED_OUT   = "TIMED_OUT"

	// TIMESTAMP_FORMAT represents the timestamp format used by Swarming APIs. Use
	// with time.Parse/time.Format.
	TIMESTAMP_FORMAT = "2006-01-02T15:04:05.999999"
)
View Source 
const (
	PUBSUB_FULLY_QUALIFIED_TOPIC_TMPL    = "projects/%s/topics/%s"
	PUBSUB_TOPIC_SWARMING_TASKS          = "swarming-tasks"
	PUBSUB_TOPIC_SWARMING_TASKS_INTERNAL = "swarming-tasks-internal"
	PUSH_URL_SWARMING_TASKS              = "pubsub/swarming-tasks"
)
View Source 
const (
	SWARMING_SERVER          = "chromium-swarm.appspot.com"
	SWARMING_SERVER_PRIVATE  = "chrome-swarming.appspot.com"
	LUCI_CLIENT_REPO         = "https://chromium.googlesource.com/infra/luci/client-py"
	RECOMMENDED_IO_TIMEOUT   = 20 * time.Minute
	RECOMMENDED_HARD_TIMEOUT = 1 * time.Hour
	RECOMMENDED_PRIORITY     = 90
	RECOMMENDED_EXPIRATION   = 4 * time.Hour
	// "priority 0 can only be used for terminate request"
	HIGHEST_PRIORITY = 1
	LOWEST_PRIORITY  = 255
)

Variables

View Source 
var (
	POOLS_PUBLIC  = []string{DIMENSION_POOL_VALUE_SKIA, DIMENSION_POOL_VALUE_SKIA_CT}
	POOLS_PRIVATE = []string{DIMENSION_POOL_VALUE_CT, DIMENSION_POOL_VALUE_SKIA_INTERNAL}

	TASK_STATES = []string{
		TASK_STATE_BOT_DIED,
		TASK_STATE_CANCELED,
		TASK_STATE_COMPLETED,
		TASK_STATE_EXPIRED,
		TASK_STATE_KILLED,
		TASK_STATE_PENDING,
		TASK_STATE_RUNNING,
		TASK_STATE_TIMED_OUT,
	}
)

Functions

func BotDimensionsToStringMap 

func BotDimensionsToStringMap(dims []*swarming.SwarmingRpcsStringListPair) map[string][]string

BotDimensionsToStringMap converts Swarming bot dimensions as represented in the Swarming API to a map[string][]string.

func BotDimensionsToStringSlice 

func BotDimensionsToStringSlice(dims []*swarming.SwarmingRpcsStringListPair) []string

BotDimensionsToStringSlice converts Swarming bot dimensions as represented in the Swarming API to a []string.

func Completed 

func Completed(t *swarming.SwarmingRpcsTaskRequestMetadata) (time.Time, error)

Completed returns a time.Time for the given task's started time.

func Created 

func Created(t *swarming.SwarmingRpcsTaskRequestMetadata) (time.Time, error)

Created returns a time.Time for the given task's created time.

func GetTagValue 

func GetTagValue(t *swarming.SwarmingRpcsTaskResult, tagKey string) (string, error)

GetTagValue returns the value for the given tag key from the given Swarming task.

func InitPubSub 

func InitPubSub(serverUrl, topicName, subscriberName string) error

InitPubSub ensures that the pub/sub topics and subscriptions needed by the TaskScheduler exist.

func PackageDimensions 

func PackageDimensions(dims map[string][]string) []string

PackageDimensions packages a map[string][]string of dimensions into a []string.

func PackageTags 

func PackageTags(tags map[string][]string) []string

PackageTags packages a map[string]string of tags into a []string.

func ParseDimensionFlags 

func ParseDimensionFlags(dimensionFlags *common.MultiString) (map[string]string, error)

ParseDimensionFlags parses the MultiString flag into a map[string]string.

func ParseDimensions 

func ParseDimensions(dims []string) (map[string][]string, error)

ParseDimensions parses a string slice of dimensions into a map[string][]string.

func ParseTags 

func ParseTags(tags []string) (map[string][]string, error)

ParseTags parses a string slice of tags into a map[string][]string.

func ParseTimestamp 

func ParseTimestamp(ts string) (time.Time, error)

ParseTimestamp returns a UTC time.Time for the given timestamp.

func ProcessBotsListCall 

func ProcessBotsListCall(call *swarming.BotsListCall) ([]*swarming.SwarmingRpcsBotInfo, error)

func RegisterPubSubServer 

func RegisterPubSubServer(h PubSubHandler, r *mux.Router)

RegisterPubSubServer adds handler to r that handle pub/sub push notifications.

func Started 

func Started(t *swarming.SwarmingRpcsTaskRequestMetadata) (time.Time, error)

Started returns a time.Time for the given task's started time.

func StringMapToBotDimensions 

func StringMapToBotDimensions(dims map[string][]string) []*swarming.SwarmingRpcsStringListPair

StringMapToBotDimensions converts Swarming bot dimensions from a map[string][]string to their Swarming API representation.

func StringMapToTaskDimensions 

func StringMapToTaskDimensions(dims map[string]string) []*swarming.SwarmingRpcsStringPair

StringMapToTaskDimensions converts Swarming task dimensions from a map[string]string to their Swarming API representation.

func TaskDimensionsToStringMap 

func TaskDimensionsToStringMap(dims []*swarming.SwarmingRpcsStringPair) map[string][]string

TaskDimensionsToStringMap converts Swarming task dimensions as represented in the Swarming API to a map[string][]string.

func TaskDimensionsToStringSlice 

func TaskDimensionsToStringSlice(dims []*swarming.SwarmingRpcsStringPair) []string

TaskDimensionsToStringSlice converts Swarming task dimensions as represented in the Swarming API to a []string.

Types

type ApiClient 

type ApiClient interface {
	// SwarmingService returns the underlying swarming.Service object.
	SwarmingService() *swarming.Service

	// ListBots returns a slice of swarming.SwarmingRpcsBotInfo instances
	// corresponding to the Swarming bots matching the requested dimensions.
	ListBots(dimensions map[string]string) ([]*swarming.SwarmingRpcsBotInfo, error)

	// ListFreeBots returns a slice of swarming.SwarmingRpcsBotInfo instances
	// corresponding to the free, alive, and not-quarantined bots in the
	// given pool.
	ListFreeBots(pool string) ([]*swarming.SwarmingRpcsBotInfo, error)

	// ListDownBots returns a slice of swarming.SwarmingRpcsBotInfo instances
	// corresponding to the dead or quarantined bots in the given pool.
	ListDownBots(pool string) ([]*swarming.SwarmingRpcsBotInfo, error)

	// ListBotsForPool returns a slice of swarming.SwarmingRpcsBotInfo
	// instances corresponding to the Swarming bots in the given pool.
	ListBotsForPool(pool string) ([]*swarming.SwarmingRpcsBotInfo, error)

	GetStdoutOfTask(id string) (*swarming.SwarmingRpcsTaskOutput, error)

	GracefullyShutdownBot(id string) (*swarming.SwarmingRpcsTerminateResponse, error)

	// ListBotTasks returns a slice of SwarmingRpcsTaskResult that are the last
	// N tasks done by a bot. When limit is big (>100), this call is very expensive.
	ListBotTasks(botID string, limit int) ([]*swarming.SwarmingRpcsTaskResult, error)

	// ListTasks returns a slice of swarming.SwarmingRpcsTaskRequestMetadata
	// instances corresponding to the specified tags and within given time window.
	// The results will have TaskId, TaskResult, and Request fields populated.
	// Specify time.Time{} for start and end if you do not want to restrict on
	// time. Specify "" for state if you do not want to restrict on state.
	ListTasks(start, end time.Time, tags []string, state string) ([]*swarming.SwarmingRpcsTaskRequestMetadata, error)

	// ListSkiaTasks is ListTasks limited to pool:Skia.
	ListSkiaTasks(start, end time.Time) ([]*swarming.SwarmingRpcsTaskRequestMetadata, error)

	// ListTaskResults returns a slice of swarming.SwarmingRpcsTaskResult
	// instances corresponding to the specified tags and within given time window.
	// Specify time.Time{} for start and end if you do not want to restrict on
	// time. Specify "" for state if you do not want to restrict on state.
	// includePerformanceStats indicates whether or not to load performance
	// information (eg. overhead) in addition to the normal task data.
	ListTaskResults(start, end time.Time, tags []string, state string, includePerformanceStats bool) ([]*swarming.SwarmingRpcsTaskResult, error)

	// CancelTask cancels the task with the given ID.
	CancelTask(id string) error

	// TriggerTask triggers a task with the given request.
	TriggerTask(t *swarming.SwarmingRpcsNewTaskRequest) (*swarming.SwarmingRpcsTaskRequestMetadata, error)

	// RetryTask triggers a retry of the given task.
	RetryTask(t *swarming.SwarmingRpcsTaskRequestMetadata) (*swarming.SwarmingRpcsTaskRequestMetadata, error)

	// GetTask returns a swarming.SwarmingRpcsTaskResult instance
	// corresponding to the given Swarming task.
	GetTask(id string, includePerformanceStats bool) (*swarming.SwarmingRpcsTaskResult, error)

	// GetTaskMetadata returns a swarming.SwarmingRpcsTaskRequestMetadata instance
	// corresponding to the given Swarming task.
	GetTaskMetadata(id string) (*swarming.SwarmingRpcsTaskRequestMetadata, error)

	DeleteBots(bots []string) error
}

ApiClient is a Skia-specific wrapper around the Swarming API.

func NewApiClient 

func NewApiClient(c *http.Client, server string) (ApiClient, error)

NewApiClient returns an ApiClient instance which uses the given authenticated http.Client.

type MockApiClient 

type MockApiClient struct {
	mock.Mock
}

MockCommonImpl is a mock of swarming.ApiClient. All the methods are mocked using testify's mocking library.

func NewMockApiClient 

func NewMockApiClient() *MockApiClient

NewAPIClient returns a pointer to a newly created struct. We return the pointer because we want to make sure the methods on mock.Mock stay accessible, e.g. m.On()

func (*MockApiClient) CancelTask 

func (_m *MockApiClient) CancelTask(id string) error

CancelTask provides a mock function with given fields: id

func (*MockApiClient) DeleteBots 

func (_m *MockApiClient) DeleteBots(bots []string) error

DeleteBots provides a mock function with given fields: bots

func (*MockApiClient) GetStdoutOfTask 

func (_m *MockApiClient) GetStdoutOfTask(id string) (*v1.SwarmingRpcsTaskOutput, error)

GetStdoutOfTask provides a mock function with given fields: id

func (*MockApiClient) GetTask 

func (_m *MockApiClient) GetTask(id string, includePerformanceStats bool) (*v1.SwarmingRpcsTaskResult, error)

GetTask provides a mock function with given fields: id, includePerformanceStats

func (*MockApiClient) GetTaskMetadata 

func (_m *MockApiClient) GetTaskMetadata(id string) (*v1.SwarmingRpcsTaskRequestMetadata, error)

GetTaskMetadata provides a mock function with given fields: id

func (*MockApiClient) GracefullyShutdownBot 

func (_m *MockApiClient) GracefullyShutdownBot(id string) (*v1.SwarmingRpcsTerminateResponse, error)

GracefullyShutdownBot provides a mock function with given fields: id

func (*MockApiClient) ListBotTasks 

func (_m *MockApiClient) ListBotTasks(botID string, limit int) ([]*v1.SwarmingRpcsTaskResult, error)

ListBotTasks provides a mock function with given fields: botID, limit

func (*MockApiClient) ListBots 

func (_m *MockApiClient) ListBots(dimensions map[string]string) ([]*v1.SwarmingRpcsBotInfo, error)

ListBots provides a mock function with given fields: dimensions

func (*MockApiClient) ListBotsForPool 

func (_m *MockApiClient) ListBotsForPool(pool string) ([]*v1.SwarmingRpcsBotInfo, error)

ListBotsForPool provides a mock function with given fields: pool

func (*MockApiClient) ListDownBots 

func (_m *MockApiClient) ListDownBots(pool string) ([]*v1.SwarmingRpcsBotInfo, error)

ListDownBots provides a mock function with given fields: pool

func (*MockApiClient) ListFreeBots 

func (_m *MockApiClient) ListFreeBots(pool string) ([]*v1.SwarmingRpcsBotInfo, error)

ListFreeBots provides a mock function with given fields: pool

func (*MockApiClient) ListSkiaTasks 

func (_m *MockApiClient) ListSkiaTasks(start time.Time, end time.Time) ([]*v1.SwarmingRpcsTaskRequestMetadata, error)

ListSkiaTasks provides a mock function with given fields: start, end

func (*MockApiClient) ListTaskResults 

func (_m *MockApiClient) ListTaskResults(start time.Time, end time.Time, tags []string, state string, includePerformanceStats bool) ([]*v1.SwarmingRpcsTaskResult, error)

ListTaskResults provides a mock function with given fields: start, end, tags, state, includePerformanceStats

func (*MockApiClient) ListTasks 

func (_m *MockApiClient) ListTasks(start time.Time, end time.Time, tags []string, state string) ([]*v1.SwarmingRpcsTaskRequestMetadata, error)

ListTasks provides a mock function with given fields: start, end, tags, state

func (*MockApiClient) RetryTask 

func (_m *MockApiClient) RetryTask(t *v1.SwarmingRpcsTaskRequestMetadata) (*v1.SwarmingRpcsTaskRequestMetadata, error)

RetryTask provides a mock function with given fields: t

func (*MockApiClient) SwarmingService 

func (_m *MockApiClient) SwarmingService() *v1.Service

SwarmingService provides a mock function with given fields:

func (*MockApiClient) TriggerTask 

func (_m *MockApiClient) TriggerTask(t *v1.SwarmingRpcsNewTaskRequest) (*v1.SwarmingRpcsTaskRequestMetadata, error)

TriggerTask provides a mock function with given fields: t

type PubSubHandler 

type PubSubHandler interface {
	HandleSwarmingPubSub(*PubSubTaskMessage) bool
}

PubSubHandler is an interface used for handling pub/sub messages.

type PubSubRequest 

type PubSubRequest struct {
	Message      pubsub.Message `json:"message"`
	Subscription string         `json:"subscription"`
}

PubSubRequest is the format of pub/sub HTTP request body.

type PubSubTaskMessage 

type PubSubTaskMessage struct {
	SwarmingTaskId string `json:"task_id"`
	UserData       string `json:"userdata"`
}

PubSubTaskMessage is a message received from Swarming via pub/sub about a Task.

type ShardOutputFormat 

type ShardOutputFormat struct {
	ExitCode string `json:"exit_code"`
	Output   string `json:"output"`
}

type SwarmingClient 

type SwarmingClient struct {
	WorkDir string

	SwarmingPy         string
	SwarmingServer     string
	ServiceAccountJSON string
	// contains filtered or unexported fields
}

func NewSwarmingClient 

func NewSwarmingClient(ctx context.Context, workDir, swarmingServer, isolateServer, serviceAccountJSON string) (*SwarmingClient, error)

NewSwarmingClient returns an instance of Swarming populated with default values.

func (*SwarmingClient) BatchArchiveTargets 

func (s *SwarmingClient) BatchArchiveTargets(ctx context.Context, isolatedGenJSONs []string, d time.Duration) (map[string]string, error)

BatchArchiveTargets batcharchives the specified isolated.gen.json files.

func (*SwarmingClient) Cleanup 

func (s *SwarmingClient) Cleanup()

func (*SwarmingClient) CreateIsolatedGenJSON 

func (s *SwarmingClient) CreateIsolatedGenJSON(isolatePath, baseDir, osType, taskName string, extraVars map[string]string, blackList []string) (string, error)

CreateIsolatedGenJSON creates isolated.gen.json files in the work dir. They then can be passed on to BatchArchiveTargets.

func (*SwarmingClient) GetIsolateClient 

func (s *SwarmingClient) GetIsolateClient() *isolate.Client

func (*SwarmingClient) TriggerSwarmingTasks 

func (s *SwarmingClient) TriggerSwarmingTasks(ctx context.Context, tasksToHashes, dimensions, tags map[string]string, cipdPackages []string, priority int, expiration, hardTimeout, ioTimeout time.Duration, idempotent, addTaskNameAsTag bool, serviceAccount string) ([]*SwarmingTask, error)

Trigger swarming using the specified hashes and dimensions.

type SwarmingTask 

type SwarmingTask struct {
	Title          string
	IsolatedHash   string
	OutputDir      string
	Dimensions     map[string]string
	Tags           map[string]string
	CipdPackages   []string
	Priority       int
	Expiration     time.Duration
	Idempotent     bool
	ServiceAccount string
}

func (*SwarmingTask) Collect 

func (t *SwarmingTask) Collect(ctx context.Context, s *SwarmingClient) (string, string, error)

func (*SwarmingTask) Trigger 

func (t *SwarmingTask) Trigger(ctx context.Context, s *SwarmingClient, hardTimeout, ioTimeout time.Duration) error

type TaskOutputFormat 

type TaskOutputFormat struct {
	Shards []ShardOutputFormat `json:"shards"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.