gcloudcleanup

README

gcloud-cleanup

Clean That Cloud!

gcloud-cleanup takes care of cleaning up resources from the Google Compute Platform.

Status

Actively running in production.

How does it fit into the rest of the system

• Deployment: Heroku, one instance per google cloud project
• Google Cloud: We talk to the Google Compute Platform via its API
• Job-board (github): We talk to job-board via HTTP to get information about which images are known.
• Worker (github): gcloud-cleanup shares a redis instance with worker for API call rate limiting

What does it really do

Instance cleaning

gcloud-cleanup finds instances matching name filters that have existed for longer than a certain cutoff time and deletes them.

This ensures that instances that failed to terminate are cleaned up.

Relevant configuration:

• GCLOUD_CLEANUP_INSTANCE_FILTERS correspond to name filters, default name eq ^testing-gce.*.
• GCLOUD_CLEANUP_INSTANCE_MAX_AGE corresponds to cutoff time, default 3h.

Image cleaning

gcloud-cleanup queries Job-board for all known images matching name filters with infra=gce, then queries Google Cloud for all known images matching name filters, and deletes any images in the Google Cloud set that are not also in the Job-board set.

This ensures that images unknown to Job-board are cleaned up.

Relevant configuration:

• GCLOUD_CLEANUP_IMAGE_FILTERS corresponds to name filters, default name eq ^travis-ci.*.

Rate limiting

GCE is not happy if we send them a gazillion API requests. In order to prevent us from being rate limited, we throttle the amount of requests we make.

NOTE: the ./ratelimit subpacakage is a vendored copy from travis-ci/worker.

Documentation

Index

• Variables
• type CLI
  • func NewCLI(c *cli.Context) *CLI
  • func (c *CLI) Run() error

Variables

var (
	// VersionString is the git describe version set at build time
	VersionString = "?"
	// RevisionString is the git revision set at build time
	RevisionString = "?"
	// RevisionURLString is the full URL to the revision set at build time
	RevisionURLString = "?"
	// GeneratedString is the build date set at build time
	GeneratedString = "?"
	// CopyrightString is the copyright set at build time
	CopyrightString = "?"
)

var (
	Flags = []cli.Flag{
		&cli.StringFlag{
			Name:    "account-json",
			Value:   "",
			Usage:   "file path to or json blob of GCE account stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_ACCOUNT_JSON", "GOOGLE_CREDENTIALS"},
		},
		&cli.StringFlag{
			Name:    "project-id",
			Value:   "",
			Usage:   "name of GCE project",
			EnvVars: []string{"GCLOUD_CLEANUP_PROJECT_ID", "GCLOUD_PROJECT"},
		},
		&cli.DurationFlag{
			Name:    "instance-max-age",
			Value:   3 * time.Hour,
			Usage:   "max age for an instance to be considered deletable",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_MAX_AGE"},
		},
		&cli.StringSliceFlag{
			Name:    "instance-filters",
			Usage:   "filters used when fetching instances for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_INSTANCE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "image-filters",
			Usage:   "filters used when fetching images for deletion",
			EnvVars: []string{"GCLOUD_CLEANUP_IMAGE_FILTERS"},
		},
		&cli.StringSliceFlag{
			Name:    "entities",
			Usage:   "entities to clean up",
			EnvVars: []string{"GCLOUD_CLEANUP_ENTITIES"},
		},
		&cli.DurationFlag{
			Name:    "loop-sleep",
			Value:   5 * time.Minute,
			Usage:   "sleep time between loops",
			EnvVars: []string{"GCLOUD_CLEANUP_LOOP_SLEEP"},
		},
		&cli.BoolFlag{
			Name:    "once",
			Usage:   "only run once, no looping",
			EnvVars: []string{"GCLOUD_CLEANUP_ONCE"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-redis-url",
			Usage:   "URL to Redis instance to use for rate limiting",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_REDIS_URL"},
		},
		&cli.StringFlag{
			Name:    "rate-limit-prefix",
			Usage:   "prefix for the rate limit key in Redis",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_PREFIX"},
		},
		&cli.IntFlag{
			Name:    "rate-limit-max-calls",
			Value:   10,
			Usage:   "number of calls per duration to let through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_MAX_CALLS"},
		},
		&cli.DurationFlag{
			Name:    "rate-limit-duration",
			Value:   1 * time.Second,
			Usage:   "interval in which to let max-calls through to the GCE API",
			EnvVars: []string{"GCLOUD_CLEANUP_RATE_LIMIT_DURATION"},
		},
		&cli.StringFlag{
			Name:    "job-board-url",
			Value:   "http://localhost:4567",
			Usage:   "url to job-board instance for fetching registered images",
			EnvVars: []string{"GCLOUD_CLEANUP_JOB_BOARD_URL", "JOB_BOARD_URL"},
		},
		&cli.BoolFlag{
			Name:    "archive-serial",
			Usage:   "archive instance serial output before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SERIAL", "ARCHIVE_SERIAL"},
		},
		&cli.StringFlag{
			Name:    "archive-bucket",
			Value:   "gcloud-cleanup-serial-output",
			Usage:   "bucket to which instance serial output will be archived before deleting",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_BUCKET", "ARCHIVE_BUCKET"},
		},
		&cli.Int64Flag{
			Name:    "archive-sample-rate",
			Value:   1,
			Usage:   "sample rate for archiving as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_ARCHIVE_SAMPLE_RATE", "ARCHIVE_SAMPLE_RATE"},
		},
		&cli.Int64Flag{
			Name:    "opencensus-sampling-rate",
			Value:   1,
			Usage:   "sample rate for trace as an inverse fraction - for sample rate n, every nth event will be sampled",
			EnvVars: []string{"GCLOUD_CLEANUP_OPENCENSUS_SAMPLING_RATE", "OPENCENSUS_SAMPLING_RATE"},
		},
		&cli.BoolFlag{
			Name:    "debug",
			Usage:   "output more stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_DEBUG", "DEBUG"},
		},
		&cli.BoolFlag{
			Name:    "opencensus-tracing-enabled",
			Usage:   "enable tracing for gcloud-cleanup",
			EnvVars: []string{"GCLOUD_CLEANUP_OPENCENSUS_TRACING_ENABLED", "OPENCENSUS_TRACING_ENABLED"},
		},
		&cli.BoolFlag{
			Name:    "noop",
			Usage:   "don't do mutative stuff",
			EnvVars: []string{"GCLOUD_CLEANUP_NOOP", "NOOP"},
		},
		&cli.StringFlag{
			Name:    "librato-email",
			Usage:   "librato account for collecting metrics",
			EnvVars: []string{"GCLOUD_CLEANUP_LIBRATO_EMAIL", "LIBRATO_EMAIL"},
		},
		&cli.StringFlag{
			Name:    "librato-token",
			Usage:   "librato token for collecting metrics",
			EnvVars: []string{"GCLOUD_CLEANUP_LIBRATO_TOKEN", "LIBRATO_TOKEN"},
		},
		&cli.StringFlag{
			Name:    "librato-source",
			Usage:   "librato source for collecting metrics",
			EnvVars: []string{"GCLOUD_CLEANUP_LIBRATO_SOURCE", "LIBRATO_SOURCE"},
		},
	}
)

Types

type CLI

type CLI struct {
	// contains filtered or unexported fields
}

func NewCLI

func NewCLI(c *cli.Context) *CLI

func (*CLI) Run

func (c *CLI) Run() error