agent

package
v3.69.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 10, 2024 License: MIT Imports: 71 Imported by: 2

Documentation

Overview

Package agent provides the key agent components - workers, worker pool, job runner, log streamer, artifact up/downloaders, etc.

It is intended for internal use of buildkite-agent only.

Index

Constants

View Source 
const (
	ArtifactPathDelimiter    = ";"
	ArtifactFallbackMimeType = "binary/octet-stream"
)
View Source 
const (
	// BuildkiteMessageMax is the maximum length of "BUILDKITE_MESSAGE=...\0"
	// environment entry passed to bootstrap, beyond which it will be truncated
	// to avoid exceeding the system limit. Note that it includes the variable
	// name, equals sign, and null terminator.
	//
	// The true limit varies by system and may be shared with other env/argv
	// data. We'll settle on an arbitrary generous but reasonable value, and
	// adjust it if issues arise.
	//
	// macOS 10.15:    256 KiB shared by environment & argv
	// Linux 4.19:     128 KiB per k=v env
	// Windows 10:  16,384 KiB shared
	// POSIX:            4 KiB minimum shared
	BuildkiteMessageMax = 64 * 1024

	// BuildkiteMessageName is the env var name of the build/commit message.
	BuildkiteMessageName = "BUILDKITE_MESSAGE"

	VerificationBehaviourWarn  = "warn"
	VerificationBehaviourBlock = "block"
)
View Source 
const (
	SignalReasonAgentRefused      = "agent_refused"
	SignalReasonAgentStop         = "agent_stop"
	SignalReasonCancel            = "cancel"
	SignalReasonSignatureRejected = "signature_rejected"
	SignalReasonProcessRunError   = "process_run_error"
)

Variables

View Source 
var (
	ErrNoSignature        = errors.New("job had no signature to verify")
	ErrVerificationFailed = errors.New("signature verification failed")
	ErrInvalidJob         = errors.New("job does not match signed step")
)
View Source 
var ArtifactPathVariableRegex = regexp.MustCompile("\\$\\{artifact\\:path\\}")
View Source 
var ProtectedEnv = map[string]struct{}{
	"BUILDKITE_AGENT_ENDPOINT":           {},
	"BUILDKITE_AGENT_ACCESS_TOKEN":       {},
	"BUILDKITE_AGENT_DEBUG":              {},
	"BUILDKITE_AGENT_PID":                {},
	"BUILDKITE_BIN_PATH":                 {},
	"BUILDKITE_CONFIG_PATH":              {},
	"BUILDKITE_BUILD_PATH":               {},
	"BUILDKITE_GIT_MIRRORS_PATH":         {},
	"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE":  {},
	"BUILDKITE_HOOKS_PATH":               {},
	"BUILDKITE_PLUGINS_PATH":             {},
	"BUILDKITE_SSH_KEYSCAN":              {},
	"BUILDKITE_GIT_SUBMODULES":           {},
	"BUILDKITE_COMMAND_EVAL":             {},
	"BUILDKITE_PLUGINS_ENABLED":          {},
	"BUILDKITE_LOCAL_HOOKS_ENABLED":      {},
	"BUILDKITE_GIT_CLONE_FLAGS":          {},
	"BUILDKITE_GIT_FETCH_FLAGS":          {},
	"BUILDKITE_GIT_CLONE_MIRROR_FLAGS":   {},
	"BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {},
	"BUILDKITE_GIT_CLEAN_FLAGS":          {},
	"BUILDKITE_SHELL":                    {},
}

Certain env can only be set by agent configuration. We show the user a warning in the bootstrap if they use any of these at a job level.

Functions

func FetchTags 

func FetchTags(ctx context.Context, l logger.Logger, conf FetchTagsConfig) []string

FetchTags loads tags from a variety of sources

func K8sTagsFromEnv added in v3.45.0 

func K8sTagsFromEnv(envn []string) (map[string]string, error)

func NewJobRunner 

func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, conf JobRunnerConfig) (jobRunner, error)

Initializes the job runner

func NewS3Client added in v3.39.0 

func NewS3Client(l logger.Logger, bucket string) (*s3.S3, error)

func ParseArtifactoryDestination 

func ParseArtifactoryDestination(destination string) (repo string, path string)

func ParseGSDestination 

func ParseGSDestination(destination string) (name string, path string)

func ParseS3Destination 

func ParseS3Destination(destination string) (string, string)

func Register 

func Register(ctx context.Context, l logger.Logger, ac APIClient, req api.AgentRegisterRequest) (*api.AgentRegisterResponse, error)

Register takes an api.Agent and registers it with the Buildkite API and populates the result of the register call

Types

type APIClient 

type APIClient interface {
	AcceptJob(context.Context, *api.Job) (*api.Job, *api.Response, error)
	AcquireJob(context.Context, string, ...api.Header) (*api.Job, *api.Response, error)
	Annotate(context.Context, string, *api.Annotation) (*api.Response, error)
	AnnotationRemove(context.Context, string, string) (*api.Response, error)
	Config() api.Config
	Connect(context.Context) (*api.Response, error)
	CreateArtifacts(context.Context, string, *api.ArtifactBatch) (*api.ArtifactBatchCreateResponse, *api.Response, error)
	Disconnect(context.Context) (*api.Response, error)
	ExistsMetaData(context.Context, string, string, string) (*api.MetaDataExists, *api.Response, error)
	FinishJob(context.Context, *api.Job) (*api.Response, error)
	FromAgentRegisterResponse(*api.AgentRegisterResponse) *api.Client
	FromPing(*api.Ping) *api.Client
	GenerateGithubCodeAccessToken(context.Context, string, string) (string, *api.Response, error)
	GetJobState(context.Context, string) (*api.JobState, *api.Response, error)
	GetMetaData(context.Context, string, string, string) (*api.MetaData, *api.Response, error)
	GetSecret(context.Context, *api.GetSecretRequest) (*api.Secret, *api.Response, error)
	Heartbeat(context.Context) (*api.Heartbeat, *api.Response, error)
	MetaDataKeys(context.Context, string, string) ([]string, *api.Response, error)
	OIDCToken(context.Context, *api.OIDCTokenRequest) (*api.OIDCToken, *api.Response, error)
	Ping(context.Context) (*api.Ping, *api.Response, error)
	PipelineUploadStatus(context.Context, string, string, ...api.Header) (*api.PipelineUploadStatus, *api.Response, error)
	Register(context.Context, *api.AgentRegisterRequest) (*api.AgentRegisterResponse, *api.Response, error)
	SaveHeaderTimes(context.Context, string, *api.HeaderTimes) (*api.Response, error)
	SearchArtifacts(context.Context, string, *api.ArtifactSearchOptions) ([]*api.Artifact, *api.Response, error)
	SetMetaData(context.Context, string, *api.MetaData) (*api.Response, error)
	StartJob(context.Context, *api.Job) (*api.Response, error)
	StepExport(context.Context, string, *api.StepExportRequest) (*api.StepExportResponse, *api.Response, error)
	StepUpdate(context.Context, string, *api.StepUpdate) (*api.Response, error)
	UpdateArtifacts(context.Context, string, map[string]string) (*api.Response, error)
	UploadChunk(context.Context, string, *api.Chunk) (*api.Response, error)
	UploadPipeline(context.Context, string, *api.PipelineChange, ...api.Header) (*api.Response, error)
}

APIClient is an interface generated for "github.com/buildkite/agent/v3/api.Client".

type AgentConfiguration 

type AgentConfiguration struct {
	ConfigPath                  string
	BootstrapScript             string
	BuildPath                   string
	HooksPath                   string
	SocketsPath                 string
	GitMirrorsPath              string
	GitMirrorsLockTimeout       int
	GitMirrorsSkipUpdate        bool
	PluginsPath                 string
	GitCheckoutFlags            string
	GitCloneFlags               string
	GitCloneMirrorFlags         string
	GitCleanFlags               string
	GitFetchFlags               string
	GitSubmodules               bool
	AllowedRepositories         []*regexp.Regexp
	AllowedPlugins              []*regexp.Regexp
	AllowedEnvironmentVariables []*regexp.Regexp
	SSHKeyscan                  bool
	CommandEval                 bool
	PluginsEnabled              bool
	PluginValidation            bool
	LocalHooksEnabled           bool
	StrictSingleHooks           bool
	RunInPty                    bool

	SigningJWKSFile  string // Where to find the key to sign pipeline uploads with (passed through to jobs, they might be uploading pipelines)
	SigningJWKSKeyID string // The key ID to sign pipeline uploads with

	VerificationJWKS             jwk.Set // The set of keys to verify jobs with
	VerificationFailureBehaviour string  // What to do if job verification fails (one of `block` or `warn`)

	ANSITimestamps             bool
	TimestampLines             bool
	HealthCheckAddr            string
	DisconnectAfterJob         bool
	DisconnectAfterIdleTimeout int
	CancelGracePeriod          int
	SignalGracePeriod          time.Duration
	EnableJobLogTmpfile        bool
	JobLogPath                 string
	WriteJobLogsToStdout       bool
	LogFormat                  string
	Shell                      string
	Profile                    string
	RedactedVars               []string
	AcquireJob                 string
	TracingBackend             string
	TracingServiceName         string
	DisableWarningsFor         []string
}

AgentConfiguration is the run-time configuration for an agent that has been loaded from the config file and command-line params

type AgentPool 

type AgentPool struct {
	// contains filtered or unexported fields
}

AgentPool manages multiple parallel AgentWorkers

func NewAgentPool 

func NewAgentPool(workers []*AgentWorker) *AgentPool

NewAgentPool returns a new AgentPool

func (*AgentPool) Start 

func (r *AgentPool) Start(ctx context.Context) error

Start kicks off the parallel AgentWorkers and waits for them to finish

func (*AgentPool) Stop 

func (r *AgentPool) Stop(graceful bool)

type AgentWorker 

type AgentWorker struct {
	// contains filtered or unexported fields
}

func NewAgentWorker 

func NewAgentWorker(l logger.Logger, a *api.AgentRegisterResponse, m *metrics.Collector, apiClient APIClient, c AgentWorkerConfig) *AgentWorker

Creates the agent worker and initializes its API Client

func (*AgentWorker) AcceptAndRunJob 

func (a *AgentWorker) AcceptAndRunJob(ctx context.Context, job *api.Job) error

Accepts a job and runs it, only returns an error if something goes wrong

func (*AgentWorker) AcquireAndRunJob 

func (a *AgentWorker) AcquireAndRunJob(ctx context.Context, jobId string) error

AcquireAndRunJob attempts to acquire a job an run it. It will retry at after the server determined interval (from the Retry-After response header) if the job is in the waiting state. If the job is in an unassignable state, it will return an error immediately. Otherwise, it will retry every 3s for 30 s. The whole operation will timeout after 5 min.

func (*AgentWorker) Connect 

func (a *AgentWorker) Connect(ctx context.Context) error

Connects the agent to the Buildkite Agent API, retrying up to 30 times if it fails.

func (*AgentWorker) Disconnect 

func (a *AgentWorker) Disconnect(ctx context.Context) error

Disconnect notifies the Buildkite API that this agent worker/session is permanently disconnecting. Don't spend long retrying, because we want to disconnect as fast as possible.

func (*AgentWorker) Heartbeat 

func (a *AgentWorker) Heartbeat(ctx context.Context) error

Performs a heatbeat

func (*AgentWorker) Ping 

func (a *AgentWorker) Ping(ctx context.Context) (*api.Job, error)

Performs a ping that checks Buildkite for a job or action to take Returns a job, or nil if none is found

func (*AgentWorker) RunJob 

func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job) error

func (*AgentWorker) Start 

func (a *AgentWorker) Start(ctx context.Context, idleMonitor *IdleMonitor) error

Starts the agent worker

func (*AgentWorker) Stop 

func (a *AgentWorker) Stop(graceful bool)

Stops the agent from accepting new work and cancels any current work it's running

type AgentWorkerConfig 

type AgentWorkerConfig struct {
	// Whether to set debug in the job
	Debug bool

	// Whether to set debugHTTP in the job
	DebugHTTP bool

	// What signal to use for worker cancellation
	CancelSignal process.Signal

	// Time wait between sending the CancelSignal and SIGKILL to the process
	// groups that the executor starts
	SignalGracePeriod time.Duration

	// The index of this agent worker
	SpawnIndex int

	// The configuration of the agent from the CLI
	AgentConfiguration AgentConfiguration

	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
	AgentStdout io.Writer
}

type ArtifactBatchCreator 

type ArtifactBatchCreator struct {
	// contains filtered or unexported fields
}

func NewArtifactBatchCreator 

func NewArtifactBatchCreator(l logger.Logger, ac APIClient, c ArtifactBatchCreatorConfig) *ArtifactBatchCreator

func (*ArtifactBatchCreator) Create 

func (a *ArtifactBatchCreator) Create(ctx context.Context) ([]*api.Artifact, error)

type ArtifactBatchCreatorConfig 

type ArtifactBatchCreatorConfig struct {
	// The ID of the Job that these artifacts belong to
	JobID string

	// All the artifacts that need to be created
	Artifacts []*api.Artifact

	// Where the artifacts are being uploaded to on the command line
	UploadDestination string

	// CreateArtifactsTimeout, sets a context.WithTimeout around the CreateArtifacts API.
	// If it's zero, there's no context timeout and the default HTTP timeout will prevail.
	CreateArtifactsTimeout time.Duration
}

type ArtifactDownloader 

type ArtifactDownloader struct {
	// contains filtered or unexported fields
}

func NewArtifactDownloader 

func NewArtifactDownloader(l logger.Logger, ac APIClient, c ArtifactDownloaderConfig) ArtifactDownloader

func (*ArtifactDownloader) Download 

func (a *ArtifactDownloader) Download(ctx context.Context) error

type ArtifactDownloaderConfig 

type ArtifactDownloaderConfig struct {
	// The ID of the Build
	BuildID string

	// The query used to find the artifacts
	Query string

	// Which step should we look at for the jobs
	Step string

	// Whether to include artifacts from retried jobs in the search
	IncludeRetriedJobs bool

	// Where we'll be downloading artifacts to
	Destination string

	// Whether to show HTTP debugging
	DebugHTTP bool
}

type ArtifactSearcher 

type ArtifactSearcher struct {
	// contains filtered or unexported fields
}

func NewArtifactSearcher 

func NewArtifactSearcher(l logger.Logger, ac APIClient, buildID string) *ArtifactSearcher

func (*ArtifactSearcher) Search 

func (a *ArtifactSearcher) Search(ctx context.Context, query, scope string, includeRetriedJobs, includeDuplicates bool) ([]*api.Artifact, error)

type ArtifactUploader 

type ArtifactUploader struct {
	// contains filtered or unexported fields
}

func NewArtifactUploader 

func NewArtifactUploader(l logger.Logger, ac APIClient, c ArtifactUploaderConfig) *ArtifactUploader

func (*ArtifactUploader) Collect 

func (a *ArtifactUploader) Collect(ctx context.Context) ([]*api.Artifact, error)

func (*ArtifactUploader) Upload 

func (a *ArtifactUploader) Upload(ctx context.Context) error

type ArtifactUploaderConfig 

type ArtifactUploaderConfig struct {
	// The ID of the Job
	JobID string

	// The path of the uploads
	Paths string

	// Where we'll be uploading artifacts
	Destination string

	// A specific Content-Type to use for all artifacts
	ContentType string

	// Whether to show HTTP debugging
	DebugHTTP bool

	// Whether to follow symbolic links when resolving globs
	GlobResolveFollowSymlinks bool

	// Whether to not upload symlinks
	UploadSkipSymlinks bool
}

type ArtifactoryDownloader 

type ArtifactoryDownloader struct {
	// contains filtered or unexported fields
}

func NewArtifactoryDownloader 

func NewArtifactoryDownloader(l logger.Logger, c ArtifactoryDownloaderConfig) *ArtifactoryDownloader

func (ArtifactoryDownloader) RepositoryFileLocation 

func (d ArtifactoryDownloader) RepositoryFileLocation() string

func (ArtifactoryDownloader) RepositoryName 

func (d ArtifactoryDownloader) RepositoryName() string

func (ArtifactoryDownloader) RepositoryPath 

func (d ArtifactoryDownloader) RepositoryPath() string

func (ArtifactoryDownloader) Start 

func (d ArtifactoryDownloader) Start(ctx context.Context) error

type ArtifactoryDownloaderConfig 

type ArtifactoryDownloaderConfig struct {
	// The Artifactory repository name and the path, for example, rt://my-repo-name/foo/bar
	Repository string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the repo
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type ArtifactoryUploader 

type ArtifactoryUploader struct {
	// The artifactory bucket path set from the destination
	Path string

	// The artifactory bucket name set from the destination
	Repository string
	// contains filtered or unexported fields
}

func NewArtifactoryUploader 

func NewArtifactoryUploader(l logger.Logger, c ArtifactoryUploaderConfig) (*ArtifactoryUploader, error)

func (*ArtifactoryUploader) URL 

func (u *ArtifactoryUploader) URL(artifact *api.Artifact) string

func (*ArtifactoryUploader) Upload 

func (u *ArtifactoryUploader) Upload(_ context.Context, artifact *api.Artifact) error

type ArtifactoryUploaderConfig 

type ArtifactoryUploaderConfig struct {
	// The destination which includes the Artifactory bucket name and the path.
	// e.g artifactory://my-repo-name/foo/bar
	Destination string

	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type Download 

type Download struct {
	// contains filtered or unexported fields
}

func NewDownload 

func NewDownload(l logger.Logger, client *http.Client, c DownloadConfig) *Download

func (Download) Start 

func (d Download) Start(ctx context.Context) error

type DownloadConfig 

type DownloadConfig struct {
	// The actual URL to get the file from
	URL string

	// The root directory of the download
	Destination string

	// Optional Headers to append to the request
	Headers map[string]string

	// The relative path that should be preserved in the download folder
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type EC2MetaData 

type EC2MetaData struct {
}

func (EC2MetaData) Get 

func (e EC2MetaData) Get() (map[string]string, error)

func (EC2MetaData) GetPaths 

func (e EC2MetaData) GetPaths(paths map[string]string) (map[string]string, error)

Takes a map of tags and meta-data paths to get, returns a map of tags and fetched values.

type EC2Tags 

type EC2Tags struct {
}

func (EC2Tags) Get 

func (e EC2Tags) Get() (map[string]string, error)

type ECSMetadata added in v3.43.0 

type ECSMetadata struct {
}

func (ECSMetadata) Get added in v3.43.0 

func (e ECSMetadata) Get() (map[string]string, error)

type Error 

type Error struct {
	Status  int    `json:"status"`  // Error code
	Message string `json:"message"` // Message describing the error.
}

An Error reports more details on an individual error in an ErrorResponse.

type FetchTagsConfig 

type FetchTagsConfig struct {
	Tags []string

	TagsFromEC2MetaData       bool
	TagsFromEC2MetaDataPaths  []string
	TagsFromEC2Tags           bool
	TagsFromECSMetaData       bool
	TagsFromGCPMetaData       bool
	TagsFromGCPMetaDataPaths  []string
	TagsFromGCPLabels         bool
	TagsFromHost              bool
	WaitForEC2TagsTimeout     time.Duration
	WaitForEC2MetaDataTimeout time.Duration
	WaitForECSMetaDataTimeout time.Duration
	WaitForGCPLabelsTimeout   time.Duration
}

type FormUploader 

type FormUploader struct {
	// contains filtered or unexported fields
}

func NewFormUploader 

func NewFormUploader(l logger.Logger, c FormUploaderConfig) *FormUploader

func (*FormUploader) URL 

func (u *FormUploader) URL(artifact *api.Artifact) string

The FormUploader doens't specify a URL, as one is provided by Buildkite after uploading

func (*FormUploader) Upload 

func (u *FormUploader) Upload(_ context.Context, artifact *api.Artifact) error

type FormUploaderConfig 

type FormUploaderConfig struct {
	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type GCPLabels 

type GCPLabels struct{}

func (GCPLabels) Get 

func (e GCPLabels) Get(ctx context.Context) (map[string]string, error)

type GCPMetaData 

type GCPMetaData struct {
}

func (GCPMetaData) Get 

func (e GCPMetaData) Get() (map[string]string, error)

func (GCPMetaData) GetPaths 

func (e GCPMetaData) GetPaths(paths map[string]string) (map[string]string, error)

Takes a map of tags and meta-data paths to get, returns a map of tags and fetched values.

type GSDownloader 

type GSDownloader struct {
	// contains filtered or unexported fields
}

func NewGSDownloader 

func NewGSDownloader(l logger.Logger, c GSDownloaderConfig) *GSDownloader

func (GSDownloader) BucketFileLocation 

func (d GSDownloader) BucketFileLocation() string

func (GSDownloader) BucketName 

func (d GSDownloader) BucketName() string

func (GSDownloader) BucketPath 

func (d GSDownloader) BucketPath() string

func (GSDownloader) Start 

func (d GSDownloader) Start(ctx context.Context) error

type GSDownloaderConfig 

type GSDownloaderConfig struct {
	// The name of the bucket
	Bucket string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the bucket
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type GSUploader 

type GSUploader struct {
	// The gs bucket path set from the destination
	BucketPath string

	// The gs bucket name set from the destination
	BucketName string
	// contains filtered or unexported fields
}

func NewGSUploader 

func NewGSUploader(l logger.Logger, c GSUploaderConfig) (*GSUploader, error)

func (*GSUploader) URL 

func (u *GSUploader) URL(artifact *api.Artifact) string

func (*GSUploader) Upload 

func (u *GSUploader) Upload(_ context.Context, artifact *api.Artifact) error

type GSUploaderConfig 

type GSUploaderConfig struct {
	// The destination which includes the GS bucket name and the path.
	// gs://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls shoud be debugged
	DebugHTTP bool
}

type IdleMonitor 

type IdleMonitor struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewIdleMonitor 

func NewIdleMonitor(totalAgents int) *IdleMonitor

func (*IdleMonitor) Idle 

func (i *IdleMonitor) Idle() bool

func (*IdleMonitor) MarkBusy 

func (i *IdleMonitor) MarkBusy(agentUUID string)

func (*IdleMonitor) MarkIdle 

func (i *IdleMonitor) MarkIdle(agentUUID string)

type JobRunner 

type JobRunner struct {

	// How the JobRunner should respond when job verification fails (one of `block` or `warn`)
	VerificationFailureBehavior string
	// contains filtered or unexported fields
}

func (*JobRunner) Cancel 

func (r *JobRunner) Cancel() error

func (*JobRunner) CancelAndStop added in v3.21.0 

func (r *JobRunner) CancelAndStop() error

func (*JobRunner) Run 

func (r *JobRunner) Run(ctx context.Context) error

Runs the job

type JobRunnerConfig 

type JobRunnerConfig struct {
	// The configuration of the agent from the CLI
	AgentConfiguration AgentConfiguration

	// How often to check if the job has been cancelled
	JobStatusInterval time.Duration

	// The JSON Web Keyset for verifying the job
	JWKS jwk.Set

	// A scope for metrics within a job
	MetricsScope *metrics.Scope

	// The job to run
	Job *api.Job

	// What signal to use for worker cancellation
	CancelSignal process.Signal

	// Whether to set debug in the job
	Debug bool

	// Whether to set debug HTTP Requests in the job
	DebugHTTP bool

	// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
	AgentStdout io.Writer
}

type LogStreamer 

type LogStreamer struct {
	// contains filtered or unexported fields
}

LogStreamer divides job log output into chunks (Process), and log streamer workers (goroutines created by Start) receive and upload those chunks. The actual uploading is performed by the callback.

func NewLogStreamer 

func NewLogStreamer(
	agentLogger logger.Logger,
	callback func(context.Context, *LogStreamerChunk) error,
	conf LogStreamerConfig,
) *LogStreamer

NewLogStreamer creates a new instance of the log streamer.

func (*LogStreamer) FailedChunks 

func (ls *LogStreamer) FailedChunks() int

func (*LogStreamer) Process 

func (ls *LogStreamer) Process(ctx context.Context, output []byte) error

Process streams the output. It returns an error if the output data cannot be processed at all (e.g. the streamer was stopped or a hard limit was reached). Transient failures to upload logs are instead handled in the callback.

func (*LogStreamer) Start 

func (ls *LogStreamer) Start(ctx context.Context) error

Start spins up a number of log streamer workers.

func (*LogStreamer) Stop 

func (ls *LogStreamer) Stop()

Stop stops the streamer.

type LogStreamerChunk 

type LogStreamerChunk struct {
	// The contents of the chunk
	Data []byte

	// The sequence number of this chunk
	Order uint64

	// The byte offset of this chunk
	Offset uint64

	// The byte size of this chunk
	Size uint64
}

type LogStreamerConfig 

type LogStreamerConfig struct {
	// How many log streamer workers are running at any one time
	Concurrency int

	// The maximum size of each chunk
	MaxChunkSizeBytes uint64

	// The maximum size of the log
	MaxSizeBytes uint64
}

LogStreamerConfig contains configuration options for the log streamer.

type LogWriter added in v3.32.0 

type LogWriter struct {
	// contains filtered or unexported fields
}

func (LogWriter) Write added in v3.32.0 

func (w LogWriter) Write(bytes []byte) (int, error)

type PipelineUploader added in v3.44.0 

type PipelineUploader struct {
	Client         APIClient
	Change         *api.PipelineChange
	JobID          string
	RetrySleepFunc func(time.Duration)
}

PipelineUploader contains the data needed to upload a pipeline to Buildkite

func (*PipelineUploader) Upload added in v3.45.0 

func (u *PipelineUploader) Upload(ctx context.Context, l logger.Logger) error

Upload will first attempt to perform an async pipeline upload and, depending on the API's response, it will poll for the upload's status.

There are 3 "routes" that are relevant 1. Async Route: /jobs/:job_uuid/pipelines?async=true 2. Sync Route: /jobs/:job_uuid/pipelines 3. Status Route: /jobs/:job_uuid/pipelines/:upload_uuid

In this method, the agent will first upload the pipeline to the Async Route. Then, depending on the response it will behave differetly

1. The Async Route responds 202: poll the Status Route until the upload has beed "applied" 2. The Async Route responds with other 2xx: exit, the upload succeeded synchronously (possibly after retry) 3. The Async Route responds with other xxx: retry uploading the pipeline to the Async Route

Note that the Sync Route is not used by this version of the agent at all. Typically, the Aysnc Route will return 202 whether or not the pipeline upload has been processed.

However, the API has the option to treat the Async Route as if it were the Sync Route by returning a 2xx that's not a 202. This will tigger option 2. While the API currently does not do this, we want to maintain the flexbitity to do so in the future. If that is implemented, the Status Route will not be polled, and either the Async Route will be retried until a (non 202) 2xx is returned from the API, or the method will exit early with no error. This reiterates option 2.

If, during a retry loop in option 3, the API returns a 2xx that is a 202, then we assume the API changed to supporting Async Uploads between retries and option 1 will be taken.

type S3Downloader 

type S3Downloader struct {
	// contains filtered or unexported fields
}

func NewS3Downloader 

func NewS3Downloader(l logger.Logger, c S3DownloaderConfig) *S3Downloader

func (S3Downloader) BucketFileLocation 

func (d S3Downloader) BucketFileLocation() string

func (S3Downloader) BucketName 

func (d S3Downloader) BucketName() string

func (S3Downloader) BucketPath 

func (d S3Downloader) BucketPath() string

func (S3Downloader) Start 

func (d S3Downloader) Start(ctx context.Context) error

type S3DownloaderConfig 

type S3DownloaderConfig struct {
	// The client for interacting with S3
	S3Client *s3.S3

	// The S3 bucket name and the path, for example, s3://my-bucket-name/foo/bar
	S3Path string

	// The root directory of the download
	Destination string

	// The relative path that should be preserved in the download folder,
	// also its location in the bucket
	Path string

	// How many times should it retry the download before giving up
	Retries int

	// If failed responses should be dumped to the log
	DebugHTTP bool
}

type S3Uploader 

type S3Uploader struct {
	// The s3 bucket path set from the destination
	BucketPath string

	// The s3 bucket name set from the destination
	BucketName string
	// contains filtered or unexported fields
}

func NewS3Uploader 

func NewS3Uploader(l logger.Logger, c S3UploaderConfig) (*S3Uploader, error)

func (*S3Uploader) URL 

func (u *S3Uploader) URL(artifact *api.Artifact) string

func (*S3Uploader) Upload 

func (u *S3Uploader) Upload(_ context.Context, artifact *api.Artifact) error

type S3UploaderConfig 

type S3UploaderConfig struct {
	// The destination which includes the S3 bucket name and the path.
	// For example, s3://my-bucket-name/foo/bar
	Destination string

	// Whether or not HTTP calls should be debugged
	DebugHTTP bool
}

type Uploader 

type Uploader interface {
	// The Artifact.URL property is populated with what ever is returned
	// from this method prior to uploading.
	URL(*api.Artifact) string

	// The actual uploading of the file
	Upload(context.Context, *api.Artifact) error
}

Source Files

View all Source files

Directories

Path Synopsis
Package plugin provides types for managing agent plugins.
 Package plugin provides types for managing agent plugins.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.