host

package
v0.0.0-...-a9abf63 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 17, 2020 License: Apache-2.0 Imports: 40 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// Collection is the name of the MongoDB collection that stores hosts.
	Collection        = "hosts"
	VolumesCollection = "volumes"
)
View Source 
const (
	// MaxLCTInterval is the maximum amount of time that can elapse before the
	// agent is considered dead. Once it has been successfully started (e.g.
	// once an agent has been deployed from the server), the agent must
	// regularly contact the server to ensure it is still alive.
	MaxLCTInterval = 5 * time.Minute
	// MaxUncommunicativeInterval is the maximum amount of time that can elapse
	// before the agent monitor is considered dead. When the host is
	// provisioning, the agent must contact the app server within this duration
	// for the agent monitor to be alive (i.e. the agent monitor has
	// successfully started its job of starting and managing the agent). After
	// initial contact, the agent must regularly contact the server so that this
	// duration does not elapse. Otherwise, the agent monitor is considered dead
	// (because it has failed to keep an agent alive that can contact the
	// server).
	MaxUncommunicativeInterval = 3 * MaxLCTInterval

	MaxTagKeyLength   = 128
	MaxTagValueLength = 256

	ErrorParentNotFound = "Parent not found"
)
View Source 
const OutputBufferSize = 1000

Variables

View Source 
var (
	IdKey                        = bsonutil.MustHaveTag(Host{}, "Id")
	DNSKey                       = bsonutil.MustHaveTag(Host{}, "Host")
	SecretKey                    = bsonutil.MustHaveTag(Host{}, "Secret")
	UserKey                      = bsonutil.MustHaveTag(Host{}, "User")
	ServicePasswordKey           = bsonutil.MustHaveTag(Host{}, "ServicePassword")
	TagKey                       = bsonutil.MustHaveTag(Host{}, "Tag")
	DistroKey                    = bsonutil.MustHaveTag(Host{}, "Distro")
	ProviderKey                  = bsonutil.MustHaveTag(Host{}, "Provider")
	IPKey                        = bsonutil.MustHaveTag(Host{}, "IP")
	ProvisionedKey               = bsonutil.MustHaveTag(Host{}, "Provisioned")
	ProvisionTimeKey             = bsonutil.MustHaveTag(Host{}, "ProvisionTime")
	ExtIdKey                     = bsonutil.MustHaveTag(Host{}, "ExternalIdentifier")
	DisplayNameKey               = bsonutil.MustHaveTag(Host{}, "DisplayName")
	RunningTaskKey               = bsonutil.MustHaveTag(Host{}, "RunningTask")
	RunningTaskGroupKey          = bsonutil.MustHaveTag(Host{}, "RunningTaskGroup")
	RunningTaskGroupOrderKey     = bsonutil.MustHaveTag(Host{}, "RunningTaskGroupOrder")
	RunningTaskBuildVariantKey   = bsonutil.MustHaveTag(Host{}, "RunningTaskBuildVariant")
	RunningTaskVersionKey        = bsonutil.MustHaveTag(Host{}, "RunningTaskVersion")
	RunningTaskProjectKey        = bsonutil.MustHaveTag(Host{}, "RunningTaskProject")
	CreateTimeKey                = bsonutil.MustHaveTag(Host{}, "CreationTime")
	ExpirationTimeKey            = bsonutil.MustHaveTag(Host{}, "ExpirationTime")
	NoExpirationKey              = bsonutil.MustHaveTag(Host{}, "NoExpiration")
	TerminationTimeKey           = bsonutil.MustHaveTag(Host{}, "TerminationTime")
	LTCTimeKey                   = bsonutil.MustHaveTag(Host{}, "LastTaskCompletedTime")
	LTCTaskKey                   = bsonutil.MustHaveTag(Host{}, "LastTask")
	LTCGroupKey                  = bsonutil.MustHaveTag(Host{}, "LastGroup")
	LTCBVKey                     = bsonutil.MustHaveTag(Host{}, "LastBuildVariant")
	LTCVersionKey                = bsonutil.MustHaveTag(Host{}, "LastVersion")
	LTCProjectKey                = bsonutil.MustHaveTag(Host{}, "LastProject")
	StatusKey                    = bsonutil.MustHaveTag(Host{}, "Status")
	AgentRevisionKey             = bsonutil.MustHaveTag(Host{}, "AgentRevision")
	NeedsNewAgentKey             = bsonutil.MustHaveTag(Host{}, "NeedsNewAgent")
	NeedsNewAgentMonitorKey      = bsonutil.MustHaveTag(Host{}, "NeedsNewAgentMonitor")
	JasperCredentialsIDKey       = bsonutil.MustHaveTag(Host{}, "JasperCredentialsID")
	NeedsReprovisionKey          = bsonutil.MustHaveTag(Host{}, "NeedsReprovision")
	ReprovisioningLockedKey      = bsonutil.MustHaveTag(Host{}, "ReprovisioningLocked")
	StartedByKey                 = bsonutil.MustHaveTag(Host{}, "StartedBy")
	InstanceTypeKey              = bsonutil.MustHaveTag(Host{}, "InstanceType")
	VolumeTotalSizeKey           = bsonutil.MustHaveTag(Host{}, "VolumeTotalSize")
	VolumesKey                   = bsonutil.MustHaveTag(Host{}, "Volumes")
	NotificationsKey             = bsonutil.MustHaveTag(Host{}, "Notifications")
	LastCommunicationTimeKey     = bsonutil.MustHaveTag(Host{}, "LastCommunicationTime")
	UserHostKey                  = bsonutil.MustHaveTag(Host{}, "UserHost")
	ZoneKey                      = bsonutil.MustHaveTag(Host{}, "Zone")
	ProjectKey                   = bsonutil.MustHaveTag(Host{}, "Project")
	ProvisionOptionsKey          = bsonutil.MustHaveTag(Host{}, "ProvisionOptions")
	ProvisionAttemptsKey         = bsonutil.MustHaveTag(Host{}, "ProvisionAttempts")
	TaskCountKey                 = bsonutil.MustHaveTag(Host{}, "TaskCount")
	StartTimeKey                 = bsonutil.MustHaveTag(Host{}, "StartTime")
	AgentStartTimeKey            = bsonutil.MustHaveTag(Host{}, "AgentStartTime")
	ComputeCostPerHourKey        = bsonutil.MustHaveTag(Host{}, "ComputeCostPerHour")
	TotalCostKey                 = bsonutil.MustHaveTag(Host{}, "TotalCost")
	TotalIdleTimeKey             = bsonutil.MustHaveTag(Host{}, "TotalIdleTime")
	HasContainersKey             = bsonutil.MustHaveTag(Host{}, "HasContainers")
	ParentIDKey                  = bsonutil.MustHaveTag(Host{}, "ParentID")
	ContainerImagesKey           = bsonutil.MustHaveTag(Host{}, "ContainerImages")
	ContainerBuildAttempt        = bsonutil.MustHaveTag(Host{}, "ContainerBuildAttempt")
	LastContainerFinishTimeKey   = bsonutil.MustHaveTag(Host{}, "LastContainerFinishTime")
	SpawnOptionsKey              = bsonutil.MustHaveTag(Host{}, "SpawnOptions")
	ContainerPoolSettingsKey     = bsonutil.MustHaveTag(Host{}, "ContainerPoolSettings")
	InstanceTagsKey              = bsonutil.MustHaveTag(Host{}, "InstanceTags")
	SSHKeyNamesKey               = bsonutil.MustHaveTag(Host{}, "SSHKeyNames")
	HomeVolumeIDKey              = bsonutil.MustHaveTag(Host{}, "HomeVolumeID")
	PortBindingsKey              = bsonutil.MustHaveTag(Host{}, "PortBindings")
	SpawnOptionsTaskIDKey        = bsonutil.MustHaveTag(SpawnOptions{}, "TaskID")
	SpawnOptionsBuildIDKey       = bsonutil.MustHaveTag(SpawnOptions{}, "BuildID")
	SpawnOptionsTimeoutKey       = bsonutil.MustHaveTag(SpawnOptions{}, "TimeoutTeardown")
	SpawnOptionsSpawnedByTaskKey = bsonutil.MustHaveTag(SpawnOptions{}, "SpawnedByTask")
	VolumeIDKey                  = bsonutil.MustHaveTag(Volume{}, "ID")
	VolumeDisplayNameKey         = bsonutil.MustHaveTag(Volume{}, "DisplayName")
	VolumeCreatedByKey           = bsonutil.MustHaveTag(Volume{}, "CreatedBy")
	VolumeTypeKey                = bsonutil.MustHaveTag(Volume{}, "Type")
	VolumeSizeKey                = bsonutil.MustHaveTag(Volume{}, "Size")
	VolumeExpirationKey          = bsonutil.MustHaveTag(Volume{}, "Expiration")
	VolumeNoExpirationKey        = bsonutil.MustHaveTag(Volume{}, "NoExpiration")
	VolumeHostKey                = bsonutil.MustHaveTag(Volume{}, "Host")
	VolumeAttachmentIDKey        = bsonutil.MustHaveTag(VolumeAttachment{}, "VolumeID")
	VolumeDeviceNameKey          = bsonutil.MustHaveTag(VolumeAttachment{}, "DeviceName")
)
View Source 
var (
	HostsByDistroDistroIDKey          = bsonutil.MustHaveTag(IdleHostsByDistroID{}, "DistroID")
	HostsByDistroIdleHostsKey         = bsonutil.MustHaveTag(IdleHostsByDistroID{}, "IdleHosts")
	HostsByDistroRunningHostsCountKey = bsonutil.MustHaveTag(IdleHostsByDistroID{}, "RunningHostsCount")
)
View Source 
var (
	CertUserIDKey            = bsonutil.MustHaveTag(certdepot.User{}, "ID")
	CertUserCertKey          = bsonutil.MustHaveTag(certdepot.User{}, "Cert")
	CertUserPrivateKeyKey    = bsonutil.MustHaveTag(certdepot.User{}, "PrivateKey")
	CertUserCertReqKey       = bsonutil.MustHaveTag(certdepot.User{}, "CertReq")
	CertUserCertRevocListKey = bsonutil.MustHaveTag(certdepot.User{}, "CertRevocList")
	CertUserTTLKey           = bsonutil.MustHaveTag(certdepot.User{}, "TTL")
)

Constants for bson struct tags.

View Source 
var All = db.Query(struct{}{})

All is a query that returns all hosts

View Source 
var AllStatic = db.Query(
	bson.M{
		ProviderKey: evergreen.HostTypeStatic,
	})
View Source 
var IsIdle = db.Query(
	bson.M{
		RunningTaskKey: bson.M{"$exists": false},
		StatusKey:      evergreen.HostRunning,
		StartedByKey:   evergreen.User,
	},
)

IsIdle is a query that returns all running Evergreen hosts with no task.

View Source 
var IsRunningAndSpawned = db.Query(
	bson.M{
		StartedByKey: bson.M{"$ne": evergreen.User},
		StatusKey:    bson.M{"$ne": evergreen.HostTerminated},
	},
)

IsRunningAndSpawned is a query that returns all running hosts spawned by an Evergreen user.

View Source 
var IsRunningTask = db.Query(
	bson.M{
		RunningTaskKey: bson.M{"$exists": true},
		StatusKey: bson.M{
			"$ne": evergreen.HostTerminated,
		},
	},
)

IsRunningTask is a query that returns all running hosts with a running task

View Source 
var IsTerminated = db.Query(
	bson.M{
		RunningTaskKey: bson.M{"$exists": false},
		StatusKey:      evergreen.HostTerminated},
)

IsTerminated is a query that returns all hosts that are terminated (and not running a task).

View Source 
var IsUninitialized = db.Query(
	bson.M{StatusKey: evergreen.HostUninitialized},
)

IsUninitialized is a query that returns all unstarted + uninitialized Evergreen hosts.

Functions

func ByDistroIDs 

func ByDistroIDs(distroIDs ...string) bson.M

ByDistroIDs produces a query that returns all up hosts of the given distros.

func ByDistroIDsOrAliasesRunning 

func ByDistroIDsOrAliasesRunning(distroNames ...string) bson.M

ByDistroIDOrAliasesRunning returns a query that returns all hosts with matching distro IDs or aliases.

func ByDynamicWithinTime 

func ByDynamicWithinTime(startTime, endTime time.Time) db.Q

ByDynamicWithinTime is a query that returns all dynamic hosts running between a certain time and another time.

func ByExpiringBetween 

func ByExpiringBetween(lowerBound time.Time, upperBound time.Time) db.Q

ByExpiringBetween produces a query that returns any host not running tasks that will expire between the specified times.

func ById 

func ById(id string) db.Q

ById produces a query that returns a host with the given id.

func ByIds 

func ByIds(ids []string) db.Q

ByIds produces a query that returns all hosts in the given list of ids.

func ByNotMonitoredSince 

func ByNotMonitoredSince(threshold time.Time) db.Q

ByNotMonitoredSince produces a query that returns all hosts whose last reachability check was before the specified threshold, filtering out user-spawned hosts and hosts currently running tasks.

func ByRunningTaskId 

func ByRunningTaskId(taskId string) db.Q

ByRunningTaskId returns a host running the task with the given id.

func ByTaskSpec 

func ByTaskSpec(group, buildVariant, project, version string) db.Q

ByTaskSpec returns a query that finds all running hosts that are running a task with the given group, buildvariant, project, and version.

func ByUnprovisionedSince 

func ByUnprovisionedSince(threshold time.Time) db.Q

ByUnprovisionedSince produces a query that returns all hosts Evergreen never finished setting up that were created before the given time.

func ByUserWithRunningStatus 

func ByUserWithRunningStatus(user string) db.Q

ByUserWithRunningStatus produces a query that returns all running hosts for the given user id.

func ByUserWithUnterminatedStatus 

func ByUserWithUnterminatedStatus(user string) db.Q

ByUserWithUnterminatedStatus produces a query that returns all running hosts for the given user id.

func ChmodCommandWithSudo 

func ChmodCommandWithSudo(script string, sudo bool) []string

func Count 

func Count(query db.Q) (int, error)

Count returns the number of hosts that satisfy the given query.

func CountAllRunningDynamicHosts 

func CountAllRunningDynamicHosts() (int, error)

func CountNoExpirationVolumesForUser 

func CountNoExpirationVolumesForUser(userID string) (int, error)

func CountRunningHosts 

func CountRunningHosts(distroID string) (int, error)

func CountSpawnhostsWithNoExpirationByUser 

func CountSpawnhostsWithNoExpirationByUser(user string) (int, error)

CountSpawnhostsWithNoExpirationByUser returns a count of all hosts associated with a given users that are considered up and should never expire.

func CountStartedTaskHosts 

func CountStartedTaskHosts() (int, error)

func CountStartedTaskHostsForDistro 

func CountStartedTaskHostsForDistro(distroID string) (int, error)

func DecommissionHostsWithDistroId 

func DecommissionHostsWithDistroId(distroId string) error

DecommissionHostsWithDistroId marks all up hosts intended for running tasks that have a matching distro ID as decommissioned.

func FindDistroForHost 

func FindDistroForHost(hostID string) (string, error)

func FindStaleRunningTasks 

func FindStaleRunningTasks(cutoff time.Duration) ([]task.Task, error)

StateRunningTasks returns tasks documents that are currently run by a host and stale

func FindTotalVolumeSizeByUser 

func FindTotalVolumeSizeByUser(user string) (int, error)

func InsertMany 

func InsertMany(hosts []Host) error

func IsIntentHostId 

func IsIntentHostId(id string) bool

func IsLive 

func IsLive() bson.M

IsLive is a query that returns all working hosts started by Evergreen

func MakeContainersAndParents 

func MakeContainersAndParents(d distro.Distro, pool *evergreen.ContainerPool, newContainersNeeded int, hostOptions CreateOptions) ([]Host, []Host, error)

func MarkInactiveStaticHosts 

func MarkInactiveStaticHosts(activeStaticHosts []string, distroID string) error

MarkInactiveStaticHosts marks static hosts in the database as terminated provided their ids aren't contained in the passed in activeStaticHosts slice. This is called in the scheduler, and marks any static host in the system that was removed from the distro as "terminated".

Previously this oepration marked these hosts as "decommissioned," which is not a state that makes sense for static hosts.

If the distro is the empty string ("") then this operation affects all distros.

func MinTaskGroupOrderRunningByTaskSpec 

func MinTaskGroupOrderRunningByTaskSpec(group, buildVariant, project, version string) (int, error)
MinTaskGroupOrderRunningByTaskSpec returns the smallest task group order number for tasks with the

given group, buildvariant, project, and version that are running on hosts. Returns 0 in the case of missing task group order numbers or no hosts.

func NeedsAgentDeploy 

func NeedsAgentDeploy(currentTime time.Time) bson.M

NeedsAgentDeploy finds hosts which need the agent to be deployed because either they do not have an agent yet or their agents have not communicated recently.

func NeedsAgentMonitorDeploy 

func NeedsAgentMonitorDeploy(currentTime time.Time) bson.M

NeedsAgentMonitorDeploy finds hosts which do not have an agent monitor yet or which should have an agent monitor but their agent has not communicated recently.

func NeedsReprovisioningLocked 

func NeedsReprovisioningLocked(currentTime time.Time) bson.M

NeedsReprovisioningLocked finds all hosts that need their provisioning changed but their provisioning has been locked for more than the max LCT interval.

func NumHostsByTaskSpec 

func NumHostsByTaskSpec(group, buildVariant, project, version string) (int, error)

NumHostsByTaskSpec returns the number of running hosts that are running a task with the given group, buildvariant, project, and version.

func Provisioning 

func Provisioning() db.Q

Provisioning returns a query used by the hostinit process to determine hosts that are started according to the cloud provider, but have not yet been provisioned by Evergreen.

func RemoveStaleInitializing 

func RemoveStaleInitializing(distroID string) error

Removes host intents that have been been uninitialized for more than 3 minutes or spawning (but not started) for more than 15 minutes for the specified distro.

If you pass the empty string as a distroID, it will remove stale host intents for *all* distros.

func RemoveStrict 

func RemoveStrict(id string) error

RemoveStrict deletes a host and errors if the host is not found

func ShCommandWithSudo 

func ShCommandWithSudo(script string, sudo bool) []string

func ShouldDeployAgent 

func ShouldDeployAgent() db.Q

ShouldDeployAgent returns legacy hosts with NeedsNewAgent set to true and are in a state in which they can deploy agents.

func ShouldDeployAgentMonitor 

func ShouldDeployAgentMonitor() db.Q

ShouldDeployAgentMonitor returns running hosts that need a new agent monitor.

func Starting 

func Starting() db.Q

Starting returns a query that finds hosts that we do not yet know to be running.

func StartingHostsByClient 

func StartingHostsByClient(limit int) (map[ClientOptions][]Host, error)

func UnsetVolumeHost 

func UnsetVolumeHost(id string) error

func UpdateAll 

func UpdateAll(query interface{}, update interface{}) error

UpdateAll updates all hosts.

func UpdateOne 

func UpdateOne(query interface{}, update interface{}) error

UpdateOne updates one host.

func UpsertOne 

func UpsertOne(query interface{}, update interface{}) (*adb.ChangeInfo, error)

UpsertOne upserts a host.

func ValidateRDPPassword 

func ValidateRDPPassword(password string) bool

XXX: if modifying any of the password validation logic, you changes must also be ported into public/static/js/directives/directives.spawn.js

Types

type ClientOptions 

type ClientOptions struct {
	Provider string `bson:"provider"`
	Region   string `bson:"region"`
	Key      string `bson:"key"`
	Secret   string `bson:"secret"`
}

type ContainersOnParents 

type ContainersOnParents struct {
	ParentHost Host
	Containers []Host
}

func GetContainersOnParents 

func GetContainersOnParents(d distro.Distro) ([]ContainersOnParents, error)

getNumContainersOnParents returns a slice of uphost parents and their respective number of current containers currently running in order of longest expected finish time

type CreateOptions 

type CreateOptions struct {
	ProvisionOptions   *ProvisionOptions
	ExpirationDuration *time.Duration
	Region             string
	UserName           string
	UserHost           bool

	HasContainers         bool
	ParentID              string
	ContainerPoolSettings *evergreen.ContainerPool
	SpawnOptions          SpawnOptions
	DockerOptions         DockerOptions
	InstanceTags          []Tag
	InstanceType          string
	NoExpiration          bool
	IsVirtualWorkstation  bool
	IsCluster             bool
	HomeVolumeSize        int
	HomeVolumeID          string
}

CreateOptions is a struct of options that are commonly passed around when creating a new cloud host.

type DistroStats 

type DistroStats []StatsByDistro

func GetStatsByDistro 

func GetStatsByDistro() (DistroStats, error)

GetStatsByDistro returns counts of up hosts broken down by distro

func (DistroStats) CountMap 

func (d DistroStats) CountMap() map[string]int

func (DistroStats) MaxHostsExceeded 

func (d DistroStats) MaxHostsExceeded() map[string]int

func (DistroStats) TasksMap 

func (d DistroStats) TasksMap() map[string]int

type DockerOptions 

type DockerOptions struct {
	// Optional parameters to define a registry name and authentication
	RegistryName     string `mapstructure:"docker_registry_name" bson:"docker_registry_name,omitempty" json:"docker_registry_name,omitempty"`
	RegistryUsername string `mapstructure:"docker_registry_user" bson:"docker_registry_user,omitempty" json:"docker_registry_user,omitempty"`
	RegistryPassword string `mapstructure:"docker_registry_pw" bson:"docker_registry_pw,omitempty" json:"docker_registry_pw,omitempty"`

	// Image is required and specifies the image for the container.
	// This can be a URL or an image base, to be combined with a registry.
	Image string `mapstructure:"image_url" bson:"image_url,omitempty" json:"image_url,omitempty"`
	// Method is either "pull" or "import" and defines how to retrieve the image.
	Method string `mapstructure:"build_type" bson:"build_type,omitempty" json:"build_type,omitempty"`
	// Command is the command to run on the docker (if not specified, will use the default entrypoint).
	Command string `mapstructure:"command" bson:"command,omitempty" json:"command,omitempty"`
	// If PublishPorts is true, any port that's exposed in the image will be published
	PublishPorts bool `mapstructure:"publish_ports" bson:"publish_ports,omitempty" json:"publish_ports,omitempty"`
	// If extra hosts are provided,these will be added to /etc/hosts on the container (in the form of hostname:IP)
	ExtraHosts []string `mapstructure:"extra_hosts" bson:"extra_hosts,omitempty" json:"extra_hosts,omitempty"`
	// If the container is created from host create, we want to skip building the image with agent
	SkipImageBuild bool `mapstructure:"skip_build" bson:"skip_build,omitempty" json:"skip_build,omitempty"`
	// list of container environment variables KEY=VALUE
	EnvironmentVars []string `mapstructure:"environment_vars" bson:"environment_vars,omitempty" json:"environment_vars,omitempty"`
}

DockerOptions contains options for starting a container

func (*DockerOptions) FromDistroSettings 

func (opts *DockerOptions) FromDistroSettings(d distro.Distro, _ string) error

func (*DockerOptions) Validate 

func (opts *DockerOptions) Validate() error

Validate checks that the settings from the config file are sane.

type EC2ProviderSettings 

type EC2ProviderSettings struct {
	Region string `bson:"region"`
	Key    string `bson:"aws_access_key_id"`
	Secret string `bson:"aws_secret_access_key"`
}

type FinishTime 

type FinishTime struct {
	Id         string    `bson:"_id"`
	FinishTime time.Time `bson:"finish_time"`
}

FinishTime is a struct for storing pairs of host IDs and last container finish times

func AggregateLastContainerFinishTimes 

func AggregateLastContainerFinishTimes() ([]FinishTime, error)

AggregateLastContainerFinishTimes returns the latest finish time for each host with containers

type Host 

type Host struct {
	Id              string        `bson:"_id" json:"id"`
	Host            string        `bson:"host_id" json:"host"`
	User            string        `bson:"user" json:"user"`
	Secret          string        `bson:"secret" json:"secret"`
	ServicePassword string        `bson:"service_password,omitempty" json:"service_password,omitempty" mapstructure:"service_password,omitempty"`
	Tag             string        `bson:"tag" json:"tag"`
	Distro          distro.Distro `bson:"distro" json:"distro"`
	Provider        string        `bson:"host_type" json:"host_type"`
	IP              string        `bson:"ip_address" json:"ip_address"`

	// secondary (external) identifier for the host
	ExternalIdentifier string `bson:"ext_identifier" json:"ext_identifier"`
	DisplayName        string `bson:"display_name" json:"display_name"`

	// physical location of host
	Project string `bson:"project" json:"project"`
	Zone    string `bson:"zone" json:"zone"`

	// True if the app server has done all necessary host setup work (although
	// the host may need to do additional provisioning before it is running).
	Provisioned       bool      `bson:"provisioned" json:"provisioned"`
	ProvisionAttempts int       `bson:"priv_attempts" json:"provision_attempts"`
	ProvisionTime     time.Time `bson:"prov_time,omitempty" json:"prov_time,omitempty"`

	ProvisionOptions *ProvisionOptions `bson:"provision_options,omitempty" json:"provision_options,omitempty"`

	// the task that is currently running on the host
	RunningTask             string `bson:"running_task,omitempty" json:"running_task,omitempty"`
	RunningTaskBuildVariant string `bson:"running_task_bv,omitempty" json:"running_task_bv,omitempty"`
	RunningTaskVersion      string `bson:"running_task_version,omitempty" json:"running_task_version,omitempty"`
	RunningTaskProject      string `bson:"running_task_project,omitempty" json:"running_task_project,omitempty"`
	RunningTaskGroup        string `bson:"running_task_group,omitempty" json:"running_task_group,omitempty"`
	RunningTaskGroupOrder   int    `bson:"running_task_group_order,omitempty" json:"running_task_group_order,omitempty"`

	// the task the most recently finished running on the host
	LastTask         string `bson:"last_task" json:"last_task"`
	LastGroup        string `bson:"last_group,omitempty" json:"last_group,omitempty"`
	LastBuildVariant string `bson:"last_bv,omitempty" json:"last_bv,omitempty"`
	LastVersion      string `bson:"last_version,omitempty" json:"last_version,omitempty"`
	LastProject      string `bson:"last_project,omitempty" json:"last_project,omitempty"`

	// the full task struct that is running on the host (only populated by certain aggregations)
	RunningTaskFull *task.Task `bson:"task_full,omitempty" json:"task_full,omitempty"`

	ExpirationTime time.Time `bson:"expiration_time,omitempty" json:"expiration_time"`
	NoExpiration   bool      `bson:"no_expiration" json:"no_expiration"`

	// creation is when the host document was inserted to the DB, start is when it was started on the cloud provider
	CreationTime time.Time `bson:"creation_time" json:"creation_time"`
	StartTime    time.Time `bson:"start_time" json:"start_time"`
	// AgentStartTime is when the agent first initiates contact with the app
	// server.
	AgentStartTime  time.Time `bson:"agent_start_time" json:"agent_start_time"`
	TerminationTime time.Time `bson:"termination_time" json:"termination_time"`
	TaskCount       int       `bson:"task_count" json:"task_count"`

	LastTaskCompletedTime time.Time `bson:"last_task_completed_time" json:"last_task_completed_time"`
	LastCommunicationTime time.Time `bson:"last_communication" json:"last_communication"`

	Status    string `bson:"status" json:"status"`
	StartedBy string `bson:"started_by" json:"started_by"`
	// True if this host was created manually by a user (i.e. with spawnhost)
	UserHost             bool   `bson:"user_host" json:"user_host"`
	AgentRevision        string `bson:"agent_revision" json:"agent_revision"`
	NeedsNewAgent        bool   `bson:"needs_agent" json:"needs_agent"`
	NeedsNewAgentMonitor bool   `bson:"needs_agent_monitor" json:"needs_agent_monitor"`

	// NeedsReprovision is set if the host needs to be reprovisioned.
	// These fields must be unset if no provisioning is needed anymore.
	NeedsReprovision     ReprovisionType `bson:"needs_reprovision,omitempty" json:"needs_reprovision,omitempty"`
	ReprovisioningLocked bool            `bson:"reprovisioning_locked,omitempty" json:"reprovisioning_locked,omitempty"`

	// JasperCredentialsID is used to match hosts to their Jasper credentials
	// for non-legacy hosts.
	JasperCredentialsID string `bson:"jasper_credentials_id" json:"jasper_credentials_id"`

	// for ec2 dynamic hosts, the instance type requested
	InstanceType string `bson:"instance_type" json:"instance_type,omitempty"`
	// for ec2 dynamic hosts, the total size of the volumes requested, in GiB
	VolumeTotalSize int64 `bson:"volume_total_size" json:"volume_total_size,omitempty"`
	// The volumeID and device name for each volume attached to the host
	Volumes []VolumeAttachment `bson:"volumes,omitempty" json:"volumes,omitempty"`

	// stores information on expiration notifications for spawn hosts
	Notifications map[string]bool `bson:"notifications,omitempty" json:"notifications,omitempty"`

	// ComputeCostPerHour is the compute (not storage) cost of one host for one hour. Cloud
	// managers can but are not required to cache this price.
	ComputeCostPerHour float64 `bson:"compute_cost_per_hour,omitempty" json:"compute_cost_per_hour,omitempty"`

	// incremented by task start and end stats collectors and
	// should reflect hosts total costs. Only populated for build-hosts
	// where host providers report costs.
	TotalCost float64 `bson:"total_cost,omitempty" json:"total_cost,omitempty"`

	// accrues the value of idle time.
	TotalIdleTime time.Duration `bson:"total_idle_time,omitempty" json:"total_idle_time,omitempty" yaml:"total_idle_time,omitempty"`

	// managed containers require different information based on host type
	// True if this host is a parent of containers
	HasContainers bool `bson:"has_containers,omitempty" json:"has_containers,omitempty"`
	// stores URLs of container images already downloaded on a parent
	ContainerImages map[string]bool `bson:"container_images,omitempty" json:"container_images,omitempty"`
	// stores the ID of the host a container is on
	ParentID string `bson:"parent_id,omitempty" json:"parent_id,omitempty"`
	// stores last expected finish time among all containers on the host
	LastContainerFinishTime time.Time `bson:"last_container_finish_time,omitempty" json:"last_container_finish_time,omitempty"`
	// ContainerPoolSettings
	ContainerPoolSettings *evergreen.ContainerPool `bson:"container_pool_settings,omitempty" json:"container_pool_settings,omitempty"`
	ContainerBuildAttempt int                      `bson:"container_build_attempt" json:"container_build_attempt"`

	// SpawnOptions holds data which the monitor uses to determine when to terminate hosts spawned by tasks.
	SpawnOptions SpawnOptions `bson:"spawn_options,omitempty" json:"spawn_options,omitempty"`

	// DockerOptions stores information for creating a container with a specific image and command
	DockerOptions DockerOptions `bson:"docker_options,omitempty" json:"docker_options,omitempty"`

	// PortBindings is populated if PublishPorts is specified when creating docker container from task
	PortBindings PortMap `bson:"port_bindings,omitempty" json:"port_bindings,omitempty"`
	// InstanceTags stores user-specified tags for instances
	InstanceTags []Tag `bson:"instance_tags,omitempty" json:"instance_tags,omitempty"`

	// SSHKeyNames contains the names of the SSH key that have been distributed
	// to this host.
	SSHKeyNames []string `bson:"ssh_key_names,omitempty" json:"ssh_key_names,omitempty"`

	IsVirtualWorkstation bool `bson:"is_virtual_workstation" json:"is_virtual_workstation"`
	// HomeVolumeSize is the size of the home volume in GB
	HomeVolumeSize int    `bson:"home_volume_size" json:"home_volume_size"`
	HomeVolumeID   string `bson:"home_volume_id" json:"home_volume_id"`
}

func AllHostsSpawnedByTasksToTerminate 

func AllHostsSpawnedByTasksToTerminate() ([]Host, error)

AllHostsSpawnedByTasksToTerminate finds all hosts spawned by tasks that should be terminated.

func Find 

func Find(query db.Q) ([]Host, error)

Find gets all Hosts for the given query.

func FindAllHostsSpawnedByTasks 

func FindAllHostsSpawnedByTasks() ([]Host, error)

FindAllHostsSpawnedByTasks finds all running hosts spawned by the `createhost` command.

func FindAllRunningContainers 

func FindAllRunningContainers() ([]Host, error)

FindAllRunningContainers finds all the containers that are currently running

func FindAllRunningParents 

func FindAllRunningParents() ([]Host, error)

FindAllRunningParents finds all running hosts that have child containers

func FindAllRunningParentsByDistroID 

func FindAllRunningParentsByDistroID(distroID string) ([]Host, error)

FindAllRunningParentsByDistroID finds all running hosts of a given distro ID with child containers.

func FindAllRunningParentsOrdered 

func FindAllRunningParentsOrdered() ([]Host, error)

FindAllRunningParentsOrdered finds all running hosts with child containers, sorted in order of soonest to latest LastContainerFinishTime

func FindByExpiringJasperCredentials 

func FindByExpiringJasperCredentials(cutoff time.Duration) ([]Host, error)

FindByExpiringJasperCredentials finds all hosts whose Jasper service credentials will expire within the given cutoff. These hosts are marked as needing provisioning changes.

func FindByNeedsJasperRestart 

func FindByNeedsJasperRestart() ([]Host, error)

FindByNeedsJasperRestart finds all hosts that are ready and waiting to restart their Jasper service.

func FindByProvisioningAttempt 

func FindByProvisioningAttempt(attempt int) ([]Host, error)

FindByFirstProvisioningAttempt finds all hosts that have not yet attempted provisioning.

func FindByShouldConvertProvisioning 

func FindByShouldConvertProvisioning() ([]Host, error)

FindBySHouldConvertProvisioning finds all hosts that are ready and waiting to convert their provisioning type.

func FindHostWithVolume 

func FindHostWithVolume(volumeID string) (*Host, error)

FindHostWithVolume finds the host associated with the specified volume ID.

func FindHostsInRange 

func FindHostsInRange(params HostsInRangeParams) ([]Host, error)

func FindHostsSpawnedByBuild 

func FindHostsSpawnedByBuild(buildID string) ([]Host, error)

FindHostsSpawnedByBuild finds hosts spawned by the `createhost` command scoped to a given build.

func FindHostsSpawnedByTask 

func FindHostsSpawnedByTask(taskID string) ([]Host, error)

FindHostsSpawnedByTask finds hosts spawned by the `createhost` command scoped to a given task.

func FindHostsToTerminate 

func FindHostsToTerminate() ([]Host, error)

func FindOne 

func FindOne(query db.Q) (*Host, error)

FindOne gets one Host for the given query.

func FindOneByIdOrTag 

func FindOneByIdOrTag(id string) (*Host, error)

FindOneByIdOrTag finds a host where the given id is stored in either the _id or tag field. (The tag field is used for the id from the host's original intent host.)

func FindOneByJasperCredentialsID 

func FindOneByJasperCredentialsID(id string) (*Host, error)

FindByJasperCredentialsID finds a host with the given Jasper credentials ID.

func FindOneId 

func FindOneId(id string) (*Host, error)

func FindRunningHosts 

func FindRunningHosts(includeSpawnHosts bool) ([]Host, error)

FindRunningHosts is the underlying query behind the hosts page's table

func FindSpawnhostsWithNoExpirationToExtend 

func FindSpawnhostsWithNoExpirationToExtend() ([]Host, error)

FindSpawnhostsWithNoExpirationToExtend returns all hosts that are set to never expire but have their expiration time within the next day and are still up.

func FindStaticNeedsNewSSHKeys 

func FindStaticNeedsNewSSHKeys(settings *evergreen.Settings) ([]Host, error)

FindStaticNeedsNewSSHKeys finds all static hosts that do not have the same set of SSH keys as those in the global settings.

func FindTerminatedHostsRunningTasks 

func FindTerminatedHostsRunningTasks() ([]Host, error)

FindTerminatedHostsRunningTasks finds all hosts that were running tasks when they were either terminated or needed to be re-provisioned.

func FindUserDataSpawnHostsProvisioning 

func FindUserDataSpawnHostsProvisioning() ([]Host, error)

FindUserDataSpawnHostsProvisioning finds all spawn hosts that have been provisioned by the app server but are still being provisioned by user data.

func GetHostsByFromIDWithStatus 

func GetHostsByFromIDWithStatus(id, status, user string, limit int) ([]Host, error)

func NewIntent 

func NewIntent(d distro.Distro, instanceName, provider string, options CreateOptions) *Host

NewIntent creates an IntentHost using the given host settings. An IntentHost is a host that does not exist yet but is intended to be picked up by the hostinit package and started. This function takes distro information, the name of the instance, the provider of the instance and a CreateOptions and returns an IntentHost.

func (*Host) AddSSHKeyName 

func (h *Host) AddSSHKeyName(name string) error

AddSSHKeyName adds the SSH key name for the host if it doesn't already have it.

func (*Host) AddTags 

func (h *Host) AddTags(tags []Tag)

AddTags adds the specified tags to the host document, or modifies an existing tag if it can be modified. Does not allow changes to tags set by Evergreen.

func (*Host) AddVolumeToHost 

func (h *Host) AddVolumeToHost(newVolume *VolumeAttachment) error

func (*Host) AgentBinary 

func (h *Host) AgentBinary() string

AgentBinary returns the path to the evergreen agent binary.

func (*Host) AgentCommand 

func (h *Host) AgentCommand(settings *evergreen.Settings) []string

AgentCommand returns the arguments to start the agent.

func (*Host) AgentMonitorOptions 

func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create

AgentMonitorOptions assembles the input to a Jasper request to start the agent monitor.

func (*Host) CacheHostData 

func (h *Host) CacheHostData() error

func (*Host) ChangeJasperDirsOwnerCommand 

func (h *Host) ChangeJasperDirsOwnerCommand() string

changeJasperDirsOwnerCommand returns the command to ensure that the Jasper directories have proper permissions.

func (*Host) CheckUserDataStartedCommand 

func (h *Host) CheckUserDataStartedCommand() (string, error)

CheckUserDataStartedCommand checks whether user data has already run on this host. If it has, it exits. Otherwise, it creates the file marking it as started.

func (*Host) ClearRunningAndSetLastTask 

func (h *Host) ClearRunningAndSetLastTask(t *task.Task) error

ClearRunningAndSetLastTask unsets the running task on the host and updates the last task fields.

func (*Host) ClearRunningTask 

func (h *Host) ClearRunningTask() error

ClearRunningTask unsets the running task on the host.

func (*Host) ClientURL 

func (h *Host) ClientURL(settings *evergreen.Settings) string

ClientURL returns the URL used to get the latest Evergreen client version.

func (*Host) CountContainersRunningAtTime 

func (h *Host) CountContainersRunningAtTime(timestamp time.Time) (int, error)

CountContainersRunningAtTime counts how many containers were running on the given parent host at the specified time, using the host StartTime and TerminationTime fields.

func (*Host) CreateSecret 

func (h *Host) CreateSecret() error

CreateSecret generates a host secret and updates the host both locally and in the database.

func (*Host) CreateServicePassword 

func (h *Host) CreateServicePassword() error

CreateServicePassword creates the password for the host's service user.

func (*Host) CurlCommand 

func (h *Host) CurlCommand(settings *evergreen.Settings) string

CurlCommand returns the command to curl the evergreen client.

func (*Host) CurlCommandWithRetry 

func (h *Host) CurlCommandWithRetry(settings *evergreen.Settings, numRetries, maxRetrySecs int) string

CurlCommandWithRetry is the same as CurlCommand but retries the request.

func (*Host) DeleteJasperCredentials 

func (h *Host) DeleteJasperCredentials(ctx context.Context, env evergreen.Environment) error

DeleteJasperCredentials deletes the Jasper credentials for the host and updates the host both in memory and in the database.

func (*Host) DeleteTags 

func (h *Host) DeleteTags(keys []string)

DeleteTags removes tags specified by their keys, only if those keys are allowed to be deleted. Does not allow changes to tags set by Evergreen.

func (*Host) DisablePoisonedHost 

func (h *Host) DisablePoisonedHost(logs string) error

func (*Host) EstimateNumContainersForDuration 

func (h *Host) EstimateNumContainersForDuration(start, end time.Time) (float64, error)

EstimateNumberContainersForDuration estimates how many containers were running on a given host during the specified time interval by averaging the counts at the start and end. It is more accurate for shorter tasks.

func (*Host) FetchAndReinstallJasperCommands 

func (h *Host) FetchAndReinstallJasperCommands(settings *evergreen.Settings) string

FetchAndReinstallJasperCommands returns the command to fetch Jasper and restart the service with the latest version.

func (*Host) FetchJasperCommand 

func (h *Host) FetchJasperCommand(config evergreen.HostJasperConfig) string

FetchJasperCommand builds the command to download and extract the Jasper binary into the distro-specific binary directory.

func (*Host) ForceReinstallJasperCommand 

func (h *Host) ForceReinstallJasperCommand(settings *evergreen.Settings) string

ForceReinstallJasperCommand returns the command to stop the Jasper service (if it's running), delete the current Jasper service configuration (if it exists), install the new configuration, and restart the service.

func (*Host) GenerateJasperCredentials 

func (h *Host) GenerateJasperCredentials(ctx context.Context, env evergreen.Environment) (*certdepot.Credentials, error)

GenerateJasperCredentials creates the Jasper credentials for the given host without saving them to the database. If credentials already exist in the database, they are deleted.

func (*Host) GetActiveContainers 

func (h *Host) GetActiveContainers() ([]Host, error)

func (*Host) GetContainers 

func (h *Host) GetContainers() ([]Host, error)

GetContainers finds all the containers belonging to this host errors if this host is not a parent

func (*Host) GetElapsedCommunicationTime 

func (h *Host) GetElapsedCommunicationTime() time.Duration

GetElapsedCommunicationTime returns how long since this host has communicated with evergreen or vice versa

func (*Host) GetJasperProcess 

func (h *Host) GetJasperProcess(ctx context.Context, env evergreen.Environment, processID string) (complete bool, output string, err error)

GetJasperProcess makes a request to the host's Jasper service to get a started process's status. Processes with an output logger return output.

func (*Host) GetParent 

func (h *Host) GetParent() (*Host, error)

GetParent finds the parent of this container errors if host is not a container or if parent cannot be found

func (*Host) GetSSHInfo 

func (h *Host) GetSSHInfo() (*util.StaticHostInfo, error)

GetSSHInfo returns the information necessary to SSH into this host.

func (*Host) GetSSHOptions 

func (h *Host) GetSSHOptions(settings *evergreen.Settings) ([]string, error)

GetSSHOptions returns the options to SSH into this host.

func (*Host) GetTaskGroupString 

func (h *Host) GetTaskGroupString() string

func (*Host) HomeVolume 

func (h *Host) HomeVolume() *VolumeAttachment

func (*Host) HostVolumeDeviceNames 

func (h *Host) HostVolumeDeviceNames() []string

func (*Host) IdleTime 

func (h *Host) IdleTime() time.Duration

IdleTime returns how long has this host been idle

func (*Host) IncContainerBuildAttempt 

func (h *Host) IncContainerBuildAttempt() error

func (*Host) IncCost 

func (h *Host) IncCost(amt float64) error

func (*Host) IncIdleTime 

func (h *Host) IncIdleTime(dur time.Duration) error

func (*Host) IncProvisionAttempts 

func (h *Host) IncProvisionAttempts() error

func (*Host) IncTaskCount 

func (h *Host) IncTaskCount() error

func (*Host) Insert 

func (h *Host) Insert() error

func (*Host) IsContainer 

func (h *Host) IsContainer() bool

func (*Host) IsEphemeral 

func (h *Host) IsEphemeral() bool

func (*Host) IsIdleParent 

func (h *Host) IsIdleParent() (bool, error)

IsIdleParent determines whether a host has only inactive containers

func (*Host) IsSubjectToHostCreationThrottle 

func (h *Host) IsSubjectToHostCreationThrottle() bool

func (*Host) IsWaitingForAgent 

func (h *Host) IsWaitingForAgent() bool

IsWaitingForAgent provides a local predicate for the logic for whether the host needs either a new agent or agent monitor.

func (*Host) JasperBinaryFilePath 

func (h *Host) JasperBinaryFilePath(config evergreen.HostJasperConfig) string

JasperBinaryFilePath returns the full path to the Jasper binary.

func (*Host) JasperClient 

func (h *Host) JasperClient(ctx context.Context, env evergreen.Environment) (remote.Manager, error)

JasperClient returns a remote client that communicates with this host's Jasper service.

func (*Host) JasperClientCredentials 

func (h *Host) JasperClientCredentials(ctx context.Context, env evergreen.Environment) (*certdepot.Credentials, error)

JasperClientCredentials gets the Jasper credentials for a client to communicate with the host's running Jasper service. These credentials should be used only to connect to the host's Jasper service.

func (*Host) JasperCredentials 

func (h *Host) JasperCredentials(ctx context.Context, env evergreen.Environment) (*certdepot.Credentials, error)

JasperCredentials gets the Jasper credentials for this host's running Jasper service from the database. These credentials should not be used to connect to the Jasper service - use JasperClientCredentials for this purpose.

func (*Host) JasperCredentialsExpiration 

func (h *Host) JasperCredentialsExpiration(ctx context.Context, env evergreen.Environment) (time.Time, error)

JasperCredentialsExpiration returns the time at which the host's Jasper credentials will expire.

func (*Host) MakeJasperDirsCommand 

func (h *Host) MakeJasperDirsCommand() string

func (*Host) MarkAsProvisioned 

func (h *Host) MarkAsProvisioned() error

func (*Host) MarkAsReprovisioned 

func (h *Host) MarkAsReprovisioned() error

MarkAsReprovisioned indicates that the host was successfully reprovisioned.

func (*Host) MarkAsReprovisioning 

func (h *Host) MarkAsReprovisioning() error

MarkAsReprovisioning puts the host in a state that means it is going to be reprovisioned.

func (*Host) MarkReachable 

func (h *Host) MarkReachable() error

func (*Host) MarkShouldExpire 

func (h *Host) MarkShouldExpire(expireOnValue string) error

MarkShouldExpire resets a host's expiration to expire like a normal spawn host, after 24 hours.

func (*Host) MarkShouldNotExpire 

func (h *Host) MarkShouldNotExpire(expireOnValue string) error

MarkShouldNotExpire marks a host as one that should not expire and updates its expiration time to avoid early reaping.

func (*Host) MarkUserDataDoneCommands 

func (h *Host) MarkUserDataDoneCommands() (string, error)

MarkUserDataDoneCommands creates the command to make the marker file indicating user data has finished executing.

func (*Host) NeedsPortBindings 

func (h *Host) NeedsPortBindings() bool

func (*Host) PastMaxExpiration 

func (h *Host) PastMaxExpiration(extension time.Duration) error

For spawn hosts that have never been set unexpirable, this will prevent spawn hosts from being set further than 30 days. For unexpirable hosts, this will prevent them from being extended any further than the new 30 day expiration (unless it is set to unexpirable again #loophole)

func (*Host) ProvisioningUserData 

func (h *Host) ProvisioningUserData(settings *evergreen.Settings, creds *certdepot.Credentials) (*userdata.Options, error)

ProvisioningUserData creates the user data parameters to provision the host. If, for some reason, this script gets interrupted, there's no guarantee that it will succeed if run again, since we cannot enforce idempotency on the setup script.

func (*Host) QuietUninstallJasperCommand 

func (h *Host) QuietUninstallJasperCommand(config evergreen.HostJasperConfig) string

QuietUninstallJasperCommand returns the command to uninstall the Jasper service. If the service is already not installed, this no-ops.

func (*Host) Remove 

func (h *Host) Remove() error

func (*Host) RemoveVolumeFromHost 

func (h *Host) RemoveVolumeFromHost(volumeId string) error

func (*Host) Replace 

func (h *Host) Replace() error

Replace overwrites an existing host document with a new one. If no existing host is found, the new one will be inserted anyway.

func (*Host) ResetLastCommunicated 

func (h *Host) ResetLastCommunicated() error

ResetLastCommunicated sets the LastCommunicationTime to be zero.

func (*Host) RestartJasperCommand 

func (h *Host) RestartJasperCommand(config evergreen.HostJasperConfig) string

RestartJasperCommand returns the command to restart the Jasper service with the existing configuration.

func (*Host) RunJasperProcess 

func (h *Host) RunJasperProcess(ctx context.Context, env evergreen.Environment, opts *options.Create) ([]string, error)

RunJasperProcess makes a request to the host's Jasper service to create the process with the given options, wait for its completion, and returns the output from it.

func (*Host) RunSSHCommand 

func (h *Host) RunSSHCommand(ctx context.Context, cmd string) (string, error)

RunSSHCommand runs an SSH command on the host with the default SSH timeout.

func (*Host) RunSSHCommandWithTimeout 

func (h *Host) RunSSHCommandWithTimeout(ctx context.Context, cmd string, timeout time.Duration) (string, error)

RunSSHCommandWithTimeout runs an SSH command on the host with the given timeout.

func (*Host) RunSSHShellScript 

func (h *Host) RunSSHShellScript(ctx context.Context, script string, sudo bool, sudoUser string) (string, error)

RunSSHShellScript runs a shell script on a remote host over SSH with the default SSH timeout.

func (*Host) RunSSHShellScriptWithTimeout 

func (h *Host) RunSSHShellScriptWithTimeout(ctx context.Context, script string, sudo bool, sudoUser string, timeout time.Duration) (string, error)

RunSSHShellScript runs a shell script on a remote host over SSH with the given timeout.

func (*Host) SaveJasperCredentials 

func (h *Host) SaveJasperCredentials(ctx context.Context, env evergreen.Environment, creds *certdepot.Credentials) error

SaveJasperCredentials saves the given Jasper credentials in the database for the host.

func (*Host) SetAgentRevision 

func (h *Host) SetAgentRevision(agentRevision string) error

SetAgentRevision sets the updated agent revision for the host

func (*Host) SetAgentStartTime 

func (h *Host) SetAgentStartTime() error

func (*Host) SetDNSName 

func (h *Host) SetDNSName(dnsName string) error

SetDNSName updates the DNS name for a given host once

func (*Host) SetDecommissioned 

func (h *Host) SetDecommissioned(user string, logs string) error

func (*Host) SetDisplayName 

func (h *Host) SetDisplayName(newName string) error

func (*Host) SetExpirationNotification 

func (h *Host) SetExpirationNotification(thresholdKey string) error

SetExpirationNotification updates the notification time for a spawn host

func (*Host) SetExpirationTime 

func (h *Host) SetExpirationTime(expirationTime time.Time) error

SetExpirationTime updates the expiration time of a spawn host

func (*Host) SetExtId 

func (h *Host) SetExtId() error

func (*Host) SetHomeVolumeID 

func (h *Host) SetHomeVolumeID(volumeID string) error

func (*Host) SetIPv6Address 

func (h *Host) SetIPv6Address(ipv6Address string) error

func (*Host) SetInstanceType 

func (h *Host) SetInstanceType(instanceType string) error

SetInstanceType updates the host's instance type in the database.

func (*Host) SetNeedsAgentDeploy 

func (h *Host) SetNeedsAgentDeploy(needsDeploy bool) error

SetNeedsAgentDeploy indicates that the host's agent or agent monitor needs to be deployed.

func (*Host) SetNeedsJasperRestart 

func (h *Host) SetNeedsJasperRestart(user string) error

SetNeedsJasperRestart sets this host as needing to have its Jasper service restarted as long as the host does not already need a different reprovisioning change. If the host is ready to reprovision now (i.e. no agent monitor is running), it is put in the reprovisioning state.

func (*Host) SetNeedsNewAgent 

func (h *Host) SetNeedsNewAgent(needsAgent bool) error

SetNeedsNewAgent sets the "needs new agent" flag on the host.

func (*Host) SetNeedsNewAgentAtomically 

func (h *Host) SetNeedsNewAgentAtomically(needsAgent bool) error

SetNeedsNewAgentAtomically sets the "needs new agent" flag on the host atomically.

func (*Host) SetNeedsNewAgentMonitor 

func (h *Host) SetNeedsNewAgentMonitor(needsAgentMonitor bool) error

SetNeedsNewAgentMonitor sets the "needs new agent monitor" flag on the host to indicate that the host needs to have the agent monitor deployed.

func (*Host) SetNeedsNewAgentMonitorAtomically 

func (h *Host) SetNeedsNewAgentMonitorAtomically(needsAgentMonitor bool) error

SetNeedsNewAgentMonitorAtomically is the same as SetNeedsNewAgentMonitor but performs an atomic update on the host in the database.

func (*Host) SetPortMapping 

func (h *Host) SetPortMapping(portsMap PortMap) error

probably don't want to store the port mapping exactly this way

func (*Host) SetProvisionedNotRunning 

func (h *Host) SetProvisionedNotRunning() error

SetProvisionedNotRunning marks the host as having been provisioned by the app server but the host is not necessarily running yet.

func (*Host) SetProvisioning 

func (h *Host) SetProvisioning() error

SetProvisioning marks the host as initializing. Only allow this if the host is uninitialized.

func (*Host) SetQuarantined 

func (h *Host) SetQuarantined(user string, logs string) error

func (*Host) SetReprovisioningLocked 

func (h *Host) SetReprovisioningLocked(locked bool) error

SetReprovisioningLocked sets the "provisioning is locked" flag on the host to indicate that provisioning jobs should not run.

func (*Host) SetReprovisioningLockedAtomically 

func (h *Host) SetReprovisioningLockedAtomically(locked bool) error

SetReprovisioningLockedAtomically is the same as SetReprovisioningLocked but performs an atomic update on the host in the database.

func (*Host) SetRunning 

func (h *Host) SetRunning(user string) error

func (*Host) SetStatus 

func (h *Host) SetStatus(status, user string, logs string) error

func (*Host) SetStatusAtomically 

func (h *Host) SetStatusAtomically(newStatus, user string, logs string) error

SetStatusAtomically is the same as SetStatus but only updates the host if its status in the database matches currentStatus.

func (*Host) SetStopped 

func (h *Host) SetStopped(user string) error

func (*Host) SetStopping 

func (h *Host) SetStopping(user string) error

func (*Host) SetTags 

func (h *Host) SetTags() error

SetTags updates the host's instance tags in the database.

func (*Host) SetTerminated 

func (h *Host) SetTerminated(user, reason string) error

func (*Host) SetUnprovisioned 

func (h *Host) SetUnprovisioned() error

func (*Host) SetUserDataHostProvisioned 

func (h *Host) SetUserDataHostProvisioned() error

SetUserDataHostProvisioned sets the host to running if it was bootstrapped with user data but has not yet been marked as done provisioning.

func (*Host) SetVolumes 

func (h *Host) SetVolumes(volumes []VolumeAttachment) error

func (*Host) SetupCommand 

func (h *Host) SetupCommand() string

SetupCommand returns the command to run the host setup script.

func (*Host) SetupServiceUserCommands 

func (h *Host) SetupServiceUserCommands() (string, error)

SetupServiceUserCommands returns the commands to create a passwordless service user in the Administrator group in Windows.

func (*Host) SpawnHostGetTaskDataCommand 

func (h *Host) SpawnHostGetTaskDataCommand() []string

SpawnHostGetTaskDataCommand returns the command that fetches the task data for a spawn host.

func (*Host) SpawnHostPullTaskSyncCommand 

func (h *Host) SpawnHostPullTaskSyncCommand() []string

SpawnHostPullTaskSyncCommand returns the command that pulls the task sync directory for a spawn host.

func (*Host) SpawnHostSetupCommands 

func (h *Host) SpawnHostSetupCommands(settings *evergreen.Settings) (string, error)

SpawnHostSetupCommands returns the commands to handle setting up a spawn host with the evergreen binary and config file for the owner.

func (*Host) StartAgentMonitorRequest 

func (h *Host) StartAgentMonitorRequest(settings *evergreen.Settings) (string, error)

StartAgentMonitorRequest builds the Jasper client request that starts the agent monitor on the host. The host secret is created if it doesn't exist yet.

func (*Host) StartJasperProcess 

func (h *Host) StartJasperProcess(ctx context.Context, env evergreen.Environment, opts *options.Create) (string, error)

StartJasperProcess makes a request to the host's Jasper service to start a process with the given options without waiting for its completion.

func (*Host) StopAgentMonitor 

func (h *Host) StopAgentMonitor(ctx context.Context, env evergreen.Environment) error

StopAgentMonitor stops the agent monitor (if it is running) on the host via its Jasper service. On legacy hosts, this is a no-op.

func (*Host) Terminate 

func (h *Host) Terminate(user, reason string) error

func (*Host) UnmarshalBSON 

func (h *Host) UnmarshalBSON(in []byte) error

func (*Host) UpdateJasperCredentialsID 

func (h *Host) UpdateJasperCredentialsID(id string) error

UpdateJasperCredentialsID sets the ID of the host's Jasper credentials.

func (*Host) UpdateLastCommunicated 

func (h *Host) UpdateLastCommunicated() error

UpdateLastCommunicated sets the host's last communication time to the current time.

func (*Host) UpdateLastContainerFinishTime 

func (h *Host) UpdateLastContainerFinishTime(t time.Time) error

UpdateLastContainerFinishTime updates latest finish time for a host with containers

func (*Host) UpdateParentIDs 

func (h *Host) UpdateParentIDs() error

func (*Host) UpdateProvisioningToRunning 

func (h *Host) UpdateProvisioningToRunning() error

UpdateProvisioningToRunning changes the host status from provisioning to running, as well as logging that the host has finished provisioning.

func (*Host) UpdateRunningTask 

func (h *Host) UpdateRunningTask(t *task.Task) (bool, error)

UpdateRunningTask updates the running task in the host document, returns - true, nil on success - false, nil on duplicate key error, task is already assigned to another host - false, error on all other errors

func (*Host) Upsert 

func (h *Host) Upsert() (*adb.ChangeInfo, error)

func (*Host) UserDataDoneFile 

func (h *Host) UserDataDoneFile() (string, error)

UserDataDoneFile returns the path to the user data done marker file.

func (*Host) UserDataStartedFile 

func (h *Host) UserDataStartedFile() (string, error)

func (*Host) WithAgentMonitor 

func (h *Host) WithAgentMonitor(ctx context.Context, env evergreen.Environment, handleAgentMonitor func(procs []jasper.Process) error) error

WithAgentMonitor runs the given handler on all agent monitor processes running on the host.

func (*Host) WriteJasperCredentialsFilesCommands 

func (h *Host) WriteJasperCredentialsFilesCommands(splunk send.SplunkConnectionInfo, creds *certdepot.Credentials) (string, error)

WriteJasperCredentialsFilesCommands builds the command to write the Jasper credentials and Splunk credentials to files.

type HostGroup 

type HostGroup []Host

func AllActiveHosts 

func AllActiveHosts(distroID string) (HostGroup, error)

AllActiveHosts produces a HostGroup for all hosts with UpHost status as well as quarantined hosts. These do not count spawn hosts.

func (HostGroup) CountContainersOnParents 

func (hosts HostGroup) CountContainersOnParents() (int, error)

CountContainersOnParents counts how many containers are children of the given group of hosts

func (HostGroup) FindUphostContainersOnParents 

func (hosts HostGroup) FindUphostContainersOnParents() ([]Host, error)

FindUphostContainersOnParents returns the containers that are children of the given hosts

func (HostGroup) GetHostIds 

func (hosts HostGroup) GetHostIds() []string

GetHostIds returns a slice of host IDs for the given group of hosts

func (HostGroup) Stats 

func (hosts HostGroup) Stats() HostGroupStats

func (HostGroup) Uphosts 

func (hosts HostGroup) Uphosts() HostGroup

type HostGroupStats 

type HostGroupStats struct {
	Quarantined    int `bson:"quarantined" json:"quarantined" yaml:"quarantined"`
	Decommissioned int `bson:"decommissioned" json:"decommissioned" yaml:"decommissioned"`
	Idle           int `bson:"idle" json:"idle" yaml:"idle"`
	Active         int `bson:"active" json:"active" yaml:"active"`
	Provisioning   int `bson:"provisioning" json:"provisioning" yaml:"provisioning"`
	Total          int `bson:"total" json:"total" yaml:"total"`
}

type HostModifyOptions 

type HostModifyOptions struct {
	AddInstanceTags    []Tag
	DeleteInstanceTags []string
	InstanceType       string
	NoExpiration       *bool         // whether host should never expire
	AddHours           time.Duration // duration to extend expiration
	AttachVolume       string
	DetachVolume       string
	SubscriptionType   string
	NewName            string
}

type HostsInRangeParams 

type HostsInRangeParams struct {
	CreatedBefore time.Time
	CreatedAfter  time.Time
	User          string
	Distro        string
	Status        string
	Region        string
	UserSpawned   bool
}

type IdleHostsByDistroID 

type IdleHostsByDistroID struct {
	DistroID          string `bson:"distro_id"`
	IdleHosts         []Host `bson:"idle_hosts"`
	RunningHostsCount int    `bson:"running_hosts_count"`
}

func IdleEphemeralGroupedByDistroID 

func IdleEphemeralGroupedByDistroID() ([]IdleHostsByDistroID, error)

IdleEphemeralGroupedByDistroId groups and collates the following by distro.Id: - []host.Host of ephemeral hosts without containers which having no running task, ordered by {host.CreationTime: 1} - the total number of ephemeral hosts that are capable of running tasks

func (*IdleHostsByDistroID) MarshalBSON 

func (h *IdleHostsByDistroID) MarshalBSON() ([]byte, error)

func (*IdleHostsByDistroID) UnmarshalBSON 

func (h *IdleHostsByDistroID) UnmarshalBSON(in []byte) error

type InactiveHostCounts 

type InactiveHostCounts struct {
	HostType string `bson:"_id"`
	Count    int    `bson:"count"`
}

func CountInactiveHostsByProvider 

func CountInactiveHostsByProvider() ([]InactiveHostCounts, error)

type PortMap 

type PortMap map[string][]string

PortMap maps container port to the parent host ports (container port is formatted as <port>/<protocol>)

func GetPortMap 

func GetPortMap(m nat.PortMap) PortMap

type ProviderStats 

type ProviderStats []StatsByProvider

func GetProviderCounts 

func GetProviderCounts() (ProviderStats, error)

GetProvierCounts returns data on the number of hosts by different provider stats.

func (ProviderStats) Map 

func (p ProviderStats) Map() map[string]int

type ProvisionOptions 

type ProvisionOptions struct {
	// LoadCLI indicates (if set) that while provisioning the host, the CLI binary should
	// be placed onto the host after startup.
	LoadCLI bool `bson:"load_cli" json:"load_cli"`

	// TaskId if non-empty will trigger the CLI tool to fetch source and
	// artifacts for the given task.
	// Ignored if LoadCLI is false.
	TaskId string `bson:"task_id" json:"task_id"`

	// TaskSync, if set along with TaskId, will fetch the task's sync data on
	// the spawn host instead of fetching the source and artifacts. This is
	// ignored if LoadCLI is false.
	TaskSync bool `bson:"task_sync" json:"task_sync"`

	// Owner is the user associated with the host used to populate any necessary metadata.
	OwnerId string `bson:"owner_id" json:"owner_id"`
}

ProvisionOptions is struct containing options about how a new spawn host should be set up.

type ReprovisionType 

type ReprovisionType string

Reprovision represents a state change in how the host is provisioned. 

const (
	ReprovisionNone ReprovisionType = ""
	// ProvisionNew indicates a transition from legacy provisioning to
	// non-legacy provisioning.
	ReprovisionToNew ReprovisionType = "convert-to-new"
	// ReprovisionToLegacy indicates a transition from non-legacy
	// provisioning to legacy provisioning.
	ReprovisionToLegacy ReprovisionType = "convert-to-legacy"
	// ReprovisionJasperRestart indicates that the host's Jasper service should
	// restart.
	ReprovisionJasperRestart ReprovisionType = "jasper-restart"
)

Constants representing host provisioning changes.

type SpawnHostUsage 

type SpawnHostUsage struct {
	TotalHosts            int `bson:"total_hosts"`
	TotalStoppedHosts     int `bson:"total_stopped_hosts"`
	TotalUnexpirableHosts int `bson:"total_unexpirable_hosts"`
	NumUsersWithHosts     int `bson:"num_users_with_hosts"`

	TotalVolumes              int            `bson:"total_volumes"`
	TotalVolumeSize           int            `bson:"total_volume_size"`
	NumUsersWithVolumes       int            `bson:"num_users_with_volumes"`
	InstanceTypes             map[string]int `bson:"instance_types"`
	AverageComputeCostPerHour float64        `bson:"average_compute_cost_per_hour"`
}

func AggregateSpawnhostData 

func AggregateSpawnhostData() (*SpawnHostUsage, error)

type SpawnOptions 

type SpawnOptions struct {
	// TimeoutTeardown is the time that this host should be torn down. In most cases, a host
	// should be torn down due to its task or build. TimeoutTeardown is a backstop to ensure that Evergreen
	// tears down a host if a task hangs or otherwise does not finish within an expected period of time.
	TimeoutTeardown time.Time `bson:"timeout_teardown,omitempty" json:"timeout_teardown,omitempty"`

	// TimeoutTeardown is the time after which Evergreen should give up trying to set up this host.
	TimeoutSetup time.Time `bson:"timeout_setup,omitempty" json:"timeout_setup,omitempty"`

	// TaskID is the task_id of the task to which this host is pinned. When the task finishes,
	// this host should be torn down. Only one of TaskID or BuildID should be set.
	TaskID string `bson:"task_id,omitempty" json:"task_id,omitempty"`

	// BuildID is the build_id of the build to which this host is pinned. When the build finishes,
	// this host should be torn down. Only one of TaskID or BuildID should be set.
	BuildID string `bson:"build_id,omitempty" json:"build_id,omitempty"`

	// Retries is the number of times Evergreen should try to spawn this host.
	Retries int `bson:"retries,omitempty" json:"retries,omitempty"`

	// SpawnedByTask indicates that this host has been spawned by a task.
	SpawnedByTask bool `bson:"spawned_by_task,omitempty" json:"spawned_by_task,omitempty"`
}

SpawnOptions holds data which the monitor uses to determine when to terminate hosts spawned by tasks.

type StatsByDistro 

type StatsByDistro struct {
	// ID of the distro the below stats are for
	Distro string `bson:"distro" json:"distro,omitempty"`
	// Provider is the provider type of the distro
	Provider string `bson:"provider" json:"provider,omitempty"`
	// Host status that the below stats are for
	Status string `bson:"status" json:"status"`
	// Number of hosts in this status
	Count int `bson:"count" json:"count"`
	// Number of tasks running on hosts in the above group (should only be nonzero for running hosts)
	NumTasks int `bson:"num_tasks_running" json:"num_tasks_running"`
	// MaxHosts reports the pool size of the distro.
	MaxHosts int `bson:"max_hosts" json:"max_hosts"`
}

func (*StatsByDistro) MarshalBSON 

func (d *StatsByDistro) MarshalBSON() ([]byte, error)

func (*StatsByDistro) UnmarshalBSON 

func (d *StatsByDistro) UnmarshalBSON(in []byte) error

type StatsByProvider 

type StatsByProvider struct {
	// the name of a host provider
	Provider string `bson:"provider" json:"provider"`
	// Number of hosts with this provider
	Count int `bson:"count" json:"count"`
}

type Tag 

type Tag struct {
	Key           string `bson:"key" json:"key"`
	Value         string `bson:"value" json:"value"`
	CanBeModified bool   `bson:"can_be_modified" json:"can_be_modified"`
}

func MakeHostTags 

func MakeHostTags(tagSlice []string) ([]Tag, error)

MakeHostTags creates and validates a map of supplied instance tags

type Volume 

type Volume struct {
	ID               string    `bson:"_id" json:"id"`
	DisplayName      string    `bson:"display_name" json:"display_name"`
	CreatedBy        string    `bson:"created_by" json:"created_by"`
	Type             string    `bson:"type" json:"type"`
	Size             int       `bson:"size" json:"size"`
	AvailabilityZone string    `bson:"availability_zone" json:"availability_zone"`
	Expiration       time.Time `bson:"expiration" json:"expiration"`
	NoExpiration     bool      `bson:"no_expiration" json:"no_expiration"`
	CreationDate     time.Time `bson:"created_at" json:"created_at"`
	Host             string    `bson:"host,omitempty" json:"host"`
	HomeVolume       bool      `bson:"home_volume" json:"home_volume"`
}

func FindOneVolume 

func FindOneVolume(query interface{}) (*Volume, error)

FindOne gets one Volume for the given query.

func FindVolumeByID 

func FindVolumeByID(id string) (*Volume, error)

FindVolumeByID finds a volume by its ID field.

func FindVolumesByUser 

func FindVolumesByUser(userID string) ([]Volume, error)

func FindVolumesToDelete 

func FindVolumesToDelete(expirationTime time.Time) ([]Volume, error)

func FindVolumesWithNoExpirationToExtend 

func FindVolumesWithNoExpirationToExtend() ([]Volume, error)

func ValidateVolumeCanBeAttached 

func ValidateVolumeCanBeAttached(volumeID string) (*Volume, error)

func (*Volume) Insert 

func (v *Volume) Insert() error

Insert a volume into the volumes collection.

func (*Volume) Remove 

func (v *Volume) Remove() error

Remove a volume from the volumes collection. Note this shouldn't be used when you want to remove from AWS itself.

func (*Volume) SetDisplayName 

func (v *Volume) SetDisplayName(displayName string) error

func (*Volume) SetExpiration 

func (v *Volume) SetExpiration(expiration time.Time) error

func (*Volume) SetHost 

func (v *Volume) SetHost(id string) error

func (*Volume) SetNoExpiration 

func (v *Volume) SetNoExpiration(noExpiration bool) error

func (*Volume) SetSize 

func (v *Volume) SetSize(size int) error

type VolumeAttachment 

type VolumeAttachment struct {
	VolumeID   string `bson:"volume_id" json:"volume_id"`
	DeviceName string `bson:"device_name" json:"device_name"`
	IsHome     bool   `bson:"is_home" json:"is_home"`
	HostID     string `bson:"host_id" json:"host_id"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.