util

package
v0.0.0-...-03d6fc4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 23, 2019 License: BSD-3-Clause Imports: 40 Imported by: 0

Documentation

Overview

Utility to create and manage chromium builds.

Utility that contains functions to interact with ct-perf.skia.org.

Google Storage utility that contains methods for both CT master and worker scripts.

Utilities for running on a local machine.

Utility that contains methods for dealing with emails.

Utility that contains methods for both CT master and worker scripts.

Index

Constants

View Source 
const (
	// Use the CTFE proxy to Google Storage. See skbug.com/6762
	GCS_HTTP_LINK         = "https://ct.skia.org/results/"
	CT_EMAIL_DISPLAY_NAME = "Cluster Telemetry"

	GMAIL_CACHED_TOKEN = "ct_gmail_cached_token"

	// File names and dir names.
	CHROMIUM_BUILDS_DIR_NAME         = "chromium_builds"
	PAGESETS_DIR_NAME                = "page_sets"
	WEB_ARCHIVES_DIR_NAME            = "webpage_archives"
	SKPS_DIR_NAME                    = "skps"
	STORAGE_DIR_NAME                 = "storage"
	REPO_DIR_NAME                    = "skia-repo"
	TASKS_DIR_NAME                   = "tasks"
	BINARIES_DIR_NAME                = "binaries"
	LUA_TASKS_DIR_NAME               = "lua_runs"
	BENCHMARK_TASKS_DIR_NAME         = "benchmark_runs"
	PIXEL_DIFF_TASKS_DIR_NAME        = "pixel_diff_runs"
	CHROMIUM_PERF_TASKS_DIR_NAME     = "chromium_perf_runs"
	CHROMIUM_ANALYSIS_TASKS_DIR_NAME = "chromium_analysis_runs"
	FIX_ARCHIVE_TASKS_DIR_NAME       = "fix_archive_runs"
	TRACE_DOWNLOADS_DIR_NAME         = "trace_downloads"
	CHROMIUM_BUILD_ZIP_NAME          = "chromium_build.zip"

	// Limit the number of times CT tries to get a remote file before giving up.
	MAX_URI_GET_TRIES = 4

	// Pageset types supported by CT.
	PAGESET_TYPE_ALL         = "All"
	PAGESET_TYPE_100k        = "100k"
	PAGESET_TYPE_MOBILE_100k = "Mobile100k"
	PAGESET_TYPE_10k         = "10k"
	PAGESET_TYPE_MOBILE_10k  = "Mobile10k"
	PAGESET_TYPE_DUMMY_1k    = "Dummy1k" // Used for testing.

	// Names of binaries executed by CT.
	BINARY_CHROME        = "chrome"
	BINARY_RECORD_WPR    = "record_wpr"
	BINARY_RUN_BENCHMARK = "run_benchmark"
	BINARY_GCLIENT       = "gclient"
	BINARY_NINJA         = "ninja"
	BINARY_LUA_PICTURES  = "lua_pictures"
	BINARY_SKPINFO       = "skpinfo"
	BINARY_ADB           = "adb"
	BINARY_GIT           = "git"
	BINARY_MAIL          = "mail"
	BINARY_LUA           = "lua"
	// TODO(rmistry): Remove once all CT bots have been upgraded to use 2.7.11
	//                by default.
	BINARY_PYTHON_2_7_11 = "/usr/local/lib/python2.7.11/bin/python"

	// Platforms supported by CT.
	PLATFORM_ANDROID = "Android"
	PLATFORM_LINUX   = "Linux"

	// Benchmarks supported by CT.
	BENCHMARK_SKPICTURE_PRINTER        = "skpicture_printer_ct"
	BENCHMARK_RR                       = "rasterize_and_record_micro_ct"
	BENCHMARK_REPAINT                  = "repaint_ct"
	BENCHMARK_LOADING                  = "loading.cluster_telemetry"
	BENCHMARK_SCREENSHOT               = "screenshot_ct"
	BENCHMARK_RENDERING                = "rendering.cluster_telemetry"
	BENCHMARK_USECOUNTER               = "usecounter_ct"
	BENCHMARK_LEAK_DETECTION           = "leak_detection.cluster_telemetry"
	BENCHMARK_MEMORY                   = "memory.cluster_telemetry"
	BENCHMARK_METRICS_ANALYSIS         = "analysis_metrics_ct"
	BENCHMARK_V8_LOADING               = "v8.loading.cluster_telemetry"
	BENCHMARK_V8_LOADING_RUNTIME_STATS = "v8.loading_runtime_stats.cluster_telemetry"

	// Logserver link. This is only accessible from Google corp.
	MASTER_LOGSERVER_LINK = "http://uberchromegw.corp.google.com/i/skia-ct-master/"

	// Default browser args when running benchmarks.
	DEFAULT_BROWSER_ARGS = ""
	// Default value column name to use when merging CSVs.
	DEFAULT_VALUE_COLUMN_NAME = "avg"

	// Use live sites flag.
	USE_LIVE_SITES_FLAGS = "--use-live-sites"
	// Pageset repeat flag.
	PAGESET_REPEAT_FLAG = "--pageset-repeat"
	// Run Benchmark timeout flag.
	RUN_BENCHMARK_TIMEOUT_FLAG = "--run-benchmark-timeout"
	// Max pages per bot flag.
	MAX_PAGES_PER_BOT = "--max-pages-per-bot"
	// Num of retries used by analysis task.
	NUM_ANALYSIS_RETRIES = "--num-analysis-retries"

	// Defaults for custom webpages.
	DEFAULT_CUSTOM_PAGE_ARCHIVEPATH = "dummy_path"

	PKILL_TIMEOUT              = 5 * time.Minute
	HTTP_CLIENT_TIMEOUT        = 30 * time.Minute
	FETCH_GN_TIMEOUT           = 2 * time.Minute
	GN_GEN_TIMEOUT             = 2 * time.Minute
	UPDATE_DEPOT_TOOLS_TIMEOUT = 5 * time.Minute

	// util.SyncDir
	GIT_PULL_TIMEOUT     = 30 * time.Minute
	GCLIENT_SYNC_TIMEOUT = 30 * time.Minute

	// util.ResetCheckout
	GIT_BRANCH_TIMEOUT = 10 * time.Minute
	GIT_REBASE_TIMEOUT = 10 * time.Minute
	GIT_RESET_TIMEOUT  = 10 * time.Minute
	GIT_CLEAN_TIMEOUT  = 10 * time.Minute

	// util.CreateChromiumBuildOnSwarming
	SYNC_SKIA_IN_CHROME_TIMEOUT = 2 * time.Hour
	GIT_LS_REMOTE_TIMEOUT       = 5 * time.Minute
	GIT_APPLY_TIMEOUT           = 5 * time.Minute
	GN_CHROMIUM_TIMEOUT         = 30 * time.Minute
	NINJA_TIMEOUT               = 2 * time.Hour

	// util.InstallChromeAPK
	ADB_INSTALL_TIMEOUT = 15 * time.Minute

	// Capture Archives
	CAPTURE_ARCHIVES_DEFAULT_CT_BENCHMARK = "rasterize_and_record_micro_ct"

	// Capture SKPs
	REMOVE_INVALID_SKPS_TIMEOUT = 3 * time.Hour

	// Run Chromium Perf
	ADB_VERSION_TIMEOUT            = 5 * time.Minute
	ADB_ROOT_TIMEOUT               = 5 * time.Minute
	CSV_PIVOT_TABLE_MERGER_TIMEOUT = 10 * time.Minute
	CSV_MERGER_TIMEOUT             = 1 * time.Hour
	CSV_COMPARER_TIMEOUT           = 2 * time.Hour

	// Run Lua
	LUA_PICTURES_TIMEOUT   = 2 * time.Hour
	LUA_AGGREGATOR_TIMEOUT = 1 * time.Hour

	// Poller
	MAKE_ALL_TIMEOUT = 15 * time.Minute

	// Swarming constants.
	SWARMING_DIR_NAME               = "swarming"
	SWARMING_POOL                   = "CT"
	BUILD_OUTPUT_FILENAME           = "build_remote_dirs.txt"
	ISOLATE_TELEMETRY_FILENAME      = "isolate_telemetry_hash.txt"
	MAX_SWARMING_HARD_TIMEOUT_HOURS = 24
	// Timeouts.
	BATCHARCHIVE_TIMEOUT = 10 * time.Minute
	XVFB_TIMEOUT         = 5 * time.Minute
	// Isolate files.
	CREATE_PAGESETS_ISOLATE   = "create_pagesets.isolate"
	CAPTURE_ARCHIVES_ISOLATE  = "capture_archives.isolate"
	CAPTURE_SKPS_ISOLATE      = "capture_skps.isolate"
	RUN_LUA_ISOLATE           = "run_lua.isolate"
	CHROMIUM_ANALYSIS_ISOLATE = "chromium_analysis.isolate"
	CHROMIUM_PERF_ISOLATE     = "chromium_perf.isolate"
	PIXEL_DIFF_ISOLATE        = "pixel_diff.isolate"
	METRICS_ANALYSIS_ISOLATE  = "metrics_analysis.isolate"
	BUILD_REPO_ISOLATE        = "build_repo.isolate"
	ISOLATE_TELEMETRY_ISOLATE = "isolate_telemetry.isolate"
	// Swarming links and params.
	// TODO(rmistry): The below link contains "st=1262304000000" which is from 2010. This is done so
	// that swarming will not use today's timestamp as default. See if there is a better way to handle
	// this.
	SWARMING_RUN_ID_ALL_TASKS_LINK_TEMPLATE   = "https://chrome-swarming.appspot.com/tasklist?l=500&c=name&c=created_ts&c=bot&c=duration&c=state&f=runid:%s&st=1262304000000"
	SWARMING_RUN_ID_TASK_LINK_PREFIX_TEMPLATE = SWARMING_RUN_ID_ALL_TASKS_LINK_TEMPLATE + "&f=name:%s"
	// Priorities
	TASKS_PRIORITY_HIGH   = swarming.RECOMMENDED_PRIORITY
	TASKS_PRIORITY_MEDIUM = swarming.RECOMMENDED_PRIORITY + 10
	TASKS_PRIORITY_LOW    = swarming.RECOMMENDED_PRIORITY + 20

	// ct-perf.skia.org constants.
	CT_PERF_BUCKET = "cluster-telemetry-perf"
	CT_PERF_REPO   = "https://skia.googlesource.com/perf-ct"
)
View Source 
const (
	DOWNLOAD_UPLOAD_GOROUTINE_POOL_SIZE = 30
	// Use larger pool size for deletions. This is useful when deleting directories
	// with 1M/1B subdirectories from the master. Google Storage will not be overwhelmed
	// because all slaves do not do large scale deletions at the same time.
	DELETE_GOROUTINE_POOL_SIZE = 1000
)
View Source 
const (
	KEY_FILE           = "id_rsa"
	WORKER_NUM_KEYWORD = "{{worker_num}}"
)
View Source 
const (
	MAX_SYNC_TRIES = 3

	TS_FORMAT = "20060102150405"

	REMOVE_INVALID_SKPS_WORKER_POOL = 20

	MAX_SIMULTANEOUS_SWARMING_TASKS_PER_RUN = 10000

	PATCH_LIMIT = 1 << 26
)
View Source 
const (
	// Use 14 chars here instead of the traditional 7 to reduce the chances of
	// ambiguous hashes while still leaving directory lengths reasonable.
	TRUNCATED_HASH_LENGTH = 14
)

Variables

View Source 
var (
	TELEMETRY_ISOLATES_TARGET  = "telemetry_perf_tests_without_chrome"
	TELEMETRY_ISOLATES_OUT_DIR = filepath.Join("out", "telemetry_isolates")
)
View Source 
var (
	CtUser = "chrome-bot"
	// Whenever the bucket name changes, getGSLink in ctfe.js will have to be
	// updated as well.
	GCSBucketName = "cluster-telemetry"

	// Email address of cluster telemetry admins. They will be notified everytime
	// a task has started and completed.
	CtAdmins = []string{"rmistry@google.com", "benjaminwagner@google.com"}

	// Names of local directories and files.
	StorageDir             = filepath.Join("/", "b", STORAGE_DIR_NAME)
	RepoDir                = filepath.Join("/", "b", REPO_DIR_NAME)
	DepotToolsDir          = filepath.Join("/", "home", "chrome-bot", "depot_tools")
	ChromiumBuildsDir      = filepath.Join(StorageDir, CHROMIUM_BUILDS_DIR_NAME)
	ChromiumSrcDir         = filepath.Join(StorageDir, "chromium", "src")
	TelemetryBinariesDir   = filepath.Join(ChromiumSrcDir, "tools", "perf")
	TelemetrySrcDir        = filepath.Join(ChromiumSrcDir, "tools", "telemetry")
	RelativeCatapultSrcDir = filepath.Join("third_party", "catapult")
	CatapultSrcDir         = filepath.Join(ChromiumSrcDir, RelativeCatapultSrcDir)
	V8SrcDir               = filepath.Join(ChromiumSrcDir, "v8")
	TaskFileDir            = filepath.Join(StorageDir, "current_task")
	ClientSecretPath       = filepath.Join(StorageDir, "client_secret.json")
	GCSTokenPath           = filepath.Join(StorageDir, "google_storage_token.data")
	PagesetsDir            = filepath.Join(StorageDir, PAGESETS_DIR_NAME)
	WebArchivesDir         = filepath.Join(StorageDir, WEB_ARCHIVES_DIR_NAME)
	SkpsDir                = filepath.Join(StorageDir, SKPS_DIR_NAME)
	ApkName                = "ChromePublic.apk"
	SkiaTreeDir            = filepath.Join(RepoDir, "trunk")
	CtTreeDir              = filepath.Join(RepoDir, "go", "src", "go.skia.org", "infra", "ct")

	// Names of remote directories and files.
	BinariesDir             = filepath.Join(BINARIES_DIR_NAME)
	LuaRunsDir              = filepath.Join(TASKS_DIR_NAME, LUA_TASKS_DIR_NAME)
	BenchmarkRunsDir        = filepath.Join(TASKS_DIR_NAME, BENCHMARK_TASKS_DIR_NAME)
	PixelDiffRunsDir        = filepath.Join(TASKS_DIR_NAME, PIXEL_DIFF_TASKS_DIR_NAME)
	ChromiumPerfRunsDir     = filepath.Join(TASKS_DIR_NAME, CHROMIUM_PERF_TASKS_DIR_NAME)
	ChromiumAnalysisRunsDir = filepath.Join(TASKS_DIR_NAME, CHROMIUM_ANALYSIS_TASKS_DIR_NAME)
	FixArchivesRunsDir      = filepath.Join(TASKS_DIR_NAME, FIX_ARCHIVE_TASKS_DIR_NAME)
	TraceDownloadsDir       = filepath.Join(TASKS_DIR_NAME, TRACE_DOWNLOADS_DIR_NAME)

	// Information about the different CT pageset types.
	PagesetTypeToInfo = map[string]*PagesetTypeInfo{
		PAGESET_TYPE_ALL: {
			NumPages:                   1000000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1M (with desktop user-agent)",
		},
		PAGESET_TYPE_100k: {
			NumPages:                   100000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 100K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_100k: {
			NumPages:                   100000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 100K (with mobile user-agent)",
		},
		PAGESET_TYPE_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with mobile user-agent)",
		},
		PAGESET_TYPE_DUMMY_1k: {
			NumPages:                   1000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			CaptureSKPsTimeoutSecs:     300,
			PixelDiffTimeoutSecs:       300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1K (used for testing, hidden from Runs History by default)",
		},
	}

	// Frontend constants below.
	SupportedBenchmarks = []string{
		BENCHMARK_RR,
		BENCHMARK_REPAINT,
		BENCHMARK_LOADING,
		BENCHMARK_USECOUNTER,
		BENCHMARK_LEAK_DETECTION,
		BENCHMARK_RENDERING,
		BENCHMARK_MEMORY,
		BENCHMARK_V8_LOADING,
		BENCHMARK_V8_LOADING_RUNTIME_STATS,
	}

	SupportedPlatformsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (Ubuntu14.04 machines)",
		PLATFORM_ANDROID: "Android (N5X devices)",
	}

	TaskPrioritiesToDesc = map[int]string{
		TASKS_PRIORITY_HIGH:   "High priority",
		TASKS_PRIORITY_MEDIUM: "Medium priority",
		TASKS_PRIORITY_LOW:    "Low priority",
	}

	// Swarming machine dimensions.
	GCE_WORKER_DIMENSIONS          = map[string]string{"pool": SWARMING_POOL, "cores": "4"}
	GCE_ANDROID_BUILDER_DIMENSIONS = map[string]string{"pool": "CTAndroidBuilder", "cores": "64"}
	GCE_LINUX_BUILDER_DIMENSIONS   = map[string]string{"pool": "CTLinuxBuilder", "cores": "64"}
	GOLO_WORKER_DIMENSIONS         = map[string]string{"pool": SWARMING_POOL, "os": "Android"}

	// ct-perf.skia.org constants.
	CTPerfWorkDir = filepath.Join(StorageDir, "ct-perf-workdir")
)

Functions

func AddCTRunDataToPerf 

func AddCTRunDataToPerf(ctx context.Context, groupName, runID, pathToCSVResults string, gs *GcsUtil) error

AddCTRunDataToPerf converts and uploads data from the CT run to CT's perf instance.

It does the following:

  1. Adds a commit to CT Perf's synthetic repo in https://skia.googlesource.com/perf-ct/+/master
  2. Constructs a results struct in the format of https://github.com/google/skia-buildbot/blob/master/perf/FORMAT.md Ensures that the results struct has as key the runID, groupName and the git hash from (1). Populates the results struct using the output CSV file from CT's run.
  3. Create JSON file from the results struct.
  4. Uploads the results file to Google storage bucket CT_PERF_BUCKET for ingestion by ct-perf.skia.org. It is stored in location of this format: gs://<bucket>/<one or more dir names>/YYYY/MM/DD/HH/<zero or more dir names><some unique name>.json

For example, it converts the following example CSV file:

paint_op_count,traceUrls,pixels_rasterized,rasterize_time (ms),record_time_caching_disabled (ms),record_time_subsequence_caching_disabled (ms),painter_memory_usage (B),record_time_construction_disabled (ms),page_name 805.0,,1310720.0,2.449,1.128,0.283,25856.0,0.335,http://www.reuters.com (#480) 643.0,,1310720.0,2.894,0.998,0.209,24856.0,0.242,http://www.rediff.com (#490)

into 

{
  "gitHash" : "8dcc84f7dc8523dd90501a4feb1f632808337c34",
  "runID" : "rmistry-xyz",
  "key" : {
    "group_name" : "BGPT perf"
  },
  "results" : {
    "http://www.reuters.com" : {
      "default" : {
        "paint_op_count": 805.0,
        "pixels_rasterized": 1310720.0,
        "rasterize_time (ms)": 2.449,
        "record_time_caching_disabled (ms)": 1.128,
        "record_time_subsequence_caching_disabled (ms)": 0.283,
        "painter_memory_usage (B)": 25856.0,
        "record_time_construction_disabled (ms)": 0.335,
        "options" : {
          "page_rank" : 480,
        },
      },
    "http://www.rediff.com" : {
      "default" : {
        "paint_op_count": 643.0,
        "pixels_rasterized": 1310720.0,
        "rasterize_time (ms)": 2.894,
        "record_time_caching_disabled (ms)": 0.998,
        "record_time_subsequence_caching_disabled (ms)": 0.209,
        "painter_memory_usage (B)": 24856.0,
        "record_time_construction_disabled (ms)": 0.242,
        "options" : {
          "page_rank" : 490,
        },
      },
    }
  }
}

func ApplyPatch 

func ApplyPatch(ctx context.Context, patch, dir string) error

ApplyPatch applies a patch to a Git checkout.

func BuildSkiaLuaPictures 

func BuildSkiaLuaPictures(ctx context.Context, runningOnSwarming bool) (string, error)

BuildSkiaLuaPictures builds "lua_pictures" in the Skia trunk directory. Returns the directory that contains the binary.

func BuildSkiaSKPInfo 

func BuildSkiaSKPInfo(ctx context.Context, runningOnSwarming bool) (string, error)

BuildSkiaSKPInfo builds "skpinfo" in the Skia trunk directory. Returns the directory that contains the binary.

func ChromeProcessesCleaner 

func ChromeProcessesCleaner(ctx context.Context, locker sync.Locker, chromeCleanerTimer time.Duration)

Running benchmarks in parallel leads to multiple chrome instances coming up at the same time, when there are crashes chrome processes stick around which can severely impact the machine's performance. To stop this from happening chrome zombie processes are periodically killed.

func ChromiumBuildDir 

func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string

Construct the name of a directory to store a chromium build. For generic clean builds, runID should be empty.

func CleanTmpDir 

func CleanTmpDir()

CleanTmpDir deletes all tmp files from the caller because telemetry tends to generate a lot of temporary artifacts there and they take up root disk space.

func CreateChromiumBuildOnSwarming 

func CreateChromiumBuildOnSwarming(ctx context.Context, runID, targetPlatform, chromiumHash, skiaHash, pathToPyFiles string, applyPatches, uploadSingleBuild bool) (string, string, error)

runID is the unique id of the current run (typically requester + timestamp). targetPlatform is the platform the benchmark will run on (Android / Linux ). chromiumHash is the hash the checkout should be synced to. If not specified then Chromium's Tot hash is used. skiaHash is the hash the checkout should be synced to. If not specified then Skia's LKGR hash is used (the hash in Chromium's DEPS file). applyPatches if true looks for Chromium/Skia/V8/Catapult patches in the temp dir and runs once with the patch applied and once without the patch applied. uploadSingleBuild if true does not upload a 2nd build of Chromium.

func CreateCustomPagesets 

func CreateCustomPagesets(webpages []string, pagesetsDir, targetPlatform string) error

func CreateTelemetryIsolates 

func CreateTelemetryIsolates(ctx context.Context, runID, chromiumHash, pathToPyFiles string, applyPatches bool) error

CreateTelemetryIsolates creates an isolate of telemetry binaries that can be distributed to CT workers to run run_benchmark or record_wpr.

ctx is the Context to use. runID is the unique id of the current run (typically requester + timestamp). chromiumHash is the hash the checkout should be synced to. pathToPyFiles is the local path to CT's python scripts. Eg: sync_skia_in_chrome.py. applyPatches if true looks for Chromium/Skia/V8/Catapult patches in the temp dir.

func DownloadAndApplyPatch 

func DownloadAndApplyPatch(ctx context.Context, patchName, localDir, remotePatchesDir, checkout string, gs *GcsUtil) error

func DownloadPatch 

func DownloadPatch(localPath, remotePath string, gs *GcsUtil) (int64, error)

func ExecuteCmd 

func ExecuteCmd(ctx context.Context, binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer) error

ExecuteCmd calls ExecuteCmdWithConfigurableLogging with logStdout and logStderr set to true.

func ExecuteCmdWithConfigurableLogging 

func ExecuteCmdWithConfigurableLogging(ctx context.Context, binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer, logStdout, logStderr bool) error

ExecuteCmdWithConfigurableLogging executes the specified binary with the specified args and env. Stdout and Stderr are written to stdout and stderr respectively if specified. If not specified then Stdout and Stderr will be outputted only to sklog.

func GetArchivesNum 

func GetArchivesNum(gs *GcsUtil, benchmarkArgs, pagesetType string) (int, error)

GetArchivesNum returns the number of archives for the specified pagesetType. -1 is returned if USE_LIVE_SITES_FLAGS is specified or if there is an error.

func GetBasePixelDiffRemoteDir 

func GetBasePixelDiffRemoteDir(runID string) (string, error)

func GetCTPerfEmailHtml 

func GetCTPerfEmailHtml(groupName string) string

func GetChromiumHash 

func GetChromiumHash(ctx context.Context) (string, error)

GetChromiumHash uses ls-remote to find and return Chromium's Tot commit hash.

func GetCipdPackageFromAsset 

func GetCipdPackageFromAsset(assetName string) (string, error)

GetCipdPackageFromAsset returns a string of the format "path:package_name:version". It returns the latest version of the asset via gitiles.

func GetClosedChannelOfPagesets 

func GetClosedChannelOfPagesets(fileInfos []os.FileInfo) chan string

Returns channel that contains all pageset file names without the timestamp file and pyc files.

func GetCurrentTs 

func GetCurrentTs() string

func GetCustomPages 

func GetCustomPages(customWebPagesFilePath string) ([]string, error)

GetCustomPages returns the specified custom webpages. If Custom webpages are not specified then it returns an empty slice.

func GetCustomPagesWithinRange 

func GetCustomPagesWithinRange(startRange, num int, customWebpages []string) []string

func GetFailureEmailHtml 

func GetFailureEmailHtml(runID string) string

func GetHashesFromBuild 

func GetHashesFromBuild(chromiumBuild string) (string, string)

GetHashesFromBuild returns the Chromium and Skia hashes from a CT build string. Example build string: try-27af50f-d5dcd58-rmistry-20151026102511-nopatch.

func GetIntFlagValue 

func GetIntFlagValue(benchmarkArgs, flagName string, defaultValue int) int

GetIntFlagValue returns the defaultValue if the specified flag name is not in benchmarkArgs.

func GetMaxPagesPerBotValue 

func GetMaxPagesPerBotValue(benchmarkArgs string, defaultValue int) int

GetMaxPagesPerBotValue returns the defaultValue if "--max-pages-per-bot" is not specified in benchmarkArgs.

func GetNumAnalysisRetriesValue 

func GetNumAnalysisRetriesValue(benchmarkArgs string, defaultValue int) int

GetNumAnalysisRetriesValue returns the defaultValue if "--num-analysis-retries" is not specified in benchmarkArgs.

func GetNumPages 

func GetNumPages(pagesetType, customWebPagesFilePath string) (int, error)

GetNumPages returns the number of specified custom webpages. If Custom webpages are not specified then the number of pages associated with the pageset type is returned.

func GetNumPagesPerBot 

func GetNumPagesPerBot(repeatValue, maxPagesPerBot int) int

GetNumPagesPerBot returns the number of web pages each worker should process.

func GetPatchFromStorage 

func GetPatchFromStorage(patchId string) (string, error)

func GetPathToIsolates 

func GetPathToIsolates(local bool) string

GetPathToIsolates returns the location of CT's isolates.

func GetPathToPyFiles 

func GetPathToPyFiles(local, runOnMaster bool) string

GetPathToPyFiles returns the location of CT's python scripts. local should be set to true if we need the location of py files when debugging locally. runOnMaster should be set if we need the location of py files on the ct-master. If both are false then it is assumed that we are running on a swarming bot.

func GetPathToTelemetryBinaries 

func GetPathToTelemetryBinaries(local bool) string

GetPathToTelemetryBinaries returns the location of Telemetry binaries.

func GetRankFromPageset 

func GetRankFromPageset(pagesetFileName string) (int, error)

func GetRepeatValue 

func GetRepeatValue(benchmarkArgs string, defaultValue int) int

GetRepeatValue returns the defaultValue if "--pageset-repeat" is not specified in benchmarkArgs.

func GetRowsFromCSV 

func GetRowsFromCSV(csvPath string) ([]string, [][]string, error)

GetRowsFromCSV reads the provided CSV and returns it's headers (first row) and values (all other rows).

func GetRunBenchmarkOutput 

func GetRunBenchmarkOutput(b bytes.Buffer) (string, error)

func GetRunBenchmarkTimeoutValue 

func GetRunBenchmarkTimeoutValue(benchmarkArgs string, defaultValue int) int

GetRunBenchmarkTimeoutValue returns the defaultValue if "--run_benchmark_timeout" is not specified in benchmarkArgs.

func GetStartRange 

func GetStartRange(workerNum, numPagesPerBot int) int

GetStartRange returns the range worker should start processing at based on its num and how many pages it is allowed to process. 

func GetSwarmingLogsLink(runID string) string

GetSwarmingLogsLink returns HTML snippet that contains a href to the swarming logs.

func GetTimeFromTs 

func GetTimeFromTs(formattedTime string) time.Time

func InstallChromeAPK 

func InstallChromeAPK(ctx context.Context, chromiumBuildName string) error

func MailInit 

func MailInit(emailClientSecretFile, emailTokenCacheFile string) error

func MergeUploadCSVFiles 

func MergeUploadCSVFiles(ctx context.Context, runID, pathToPyFiles string, gs *GcsUtil, totalPages, maxPagesPerBot int, handleStrings bool, repeatValue int) (string, []string, error)

func MergeUploadCSVFilesOnWorkers 

func MergeUploadCSVFilesOnWorkers(ctx context.Context, localOutputDir, pathToPyFiles, runID, remoteDir, valueColumnName string, gs *GcsUtil, startRange int, handleStrings, addRanks bool, pageRankToAdditionalFields map[string]map[string]string) error

func ParseEmails 

func ParseEmails(emails string) []string

ParseEmails returns an array containing emails from the provided comma separated emails string.

func PatchesAreEmpty 

func PatchesAreEmpty(patches []string) bool

func RemoveFlagsFromArgs 

func RemoveFlagsFromArgs(benchmarkArgs string, flags ...string) string

func ResetCheckout 

func ResetCheckout(ctx context.Context, dir, resetTo string) error

ResetCheckout resets the specified Git checkout.

func ResetChromiumCheckout 

func ResetChromiumCheckout(ctx context.Context, chromiumSrcDir string) error

func RunBenchmark 

func RunBenchmark(ctx context.Context, fileInfoName, pathToPagesets, pathToPyFiles, localOutputDir, chromiumBuildName, chromiumBinary, runID, browserExtraArgs, benchmarkName, targetPlatform, benchmarkExtraArgs, pagesetType string, defaultRepeatValue int, runOnSwarming bool) (string, error)

RunBenchmark runs the specified benchmark with the specified arguments. It prints the output of the run_benchmark command and also returns the output incase the caller needs to do any post-processing on it. Incase of any errors the output will be empty.

func SavePatchToStorage 

func SavePatchToStorage(patch string) (string, error)

func SendEmail 

func SendEmail(recipients []string, subject, body string) error

SendEmail sends an email with the specified header and body to the recipients.

func SendEmailWithMarkup 

func SendEmailWithMarkup(recipients []string, subject, body, markup string) error

SendEmailWithMarkup sends an email with the specified header and body to the recipients. It also includes gmail markups. Documentation about markups supported in gmail are here: https://developers.google.com/gmail/markup/ A go-to action example is here: https://developers.google.com/gmail/markup/reference/go-to-action

func SendTaskStartEmail 

func SendTaskStartEmail(taskId int64, recipients []string, taskName, runID, description string) error

func SendTasksTerminatedEmail 

func SendTasksTerminatedEmail(recipients []string) error

SendTasksTerminatedEmail informs the recipients that their CT tasks were terminated and that they should reschedule.

func SetVarsForLocal 

func SetVarsForLocal()

func SshToBareMetalMachines 

func SshToBareMetalMachines(cmd string, workers []string, timeout time.Duration) (map[string]string, error)

SshToBareMetalMachines connects to the specified workers and runs the specified command. If the command does not complete in the given duration then all remaining workers are considered timed out. SSH also automatically substitutes the sequential number of the worker for the WORKER_NUM_KEYWORD since it is a common use case.

func SyncDir 

func SyncDir(ctx context.Context, dir string, revisions map[string]string, additionalArgs []string) error

SyncDir runs "git pull" and "gclient sync" on the specified directory. The revisions map enforces revision/hash for the solutions with the format branch@rev.

func TimeTrack 

func TimeTrack(start time.Time, name string)

func TriggerBuildRepoSwarmingTask 

func TriggerBuildRepoSwarmingTask(ctx context.Context, taskName, runID, repoAndTarget, targetPlatform, serviceAccountJSON string, hashes, patches, cipdPackages []string, singleBuild, local bool, hardTimeout, ioTimeout time.Duration) ([]string, error)

TriggerBuildRepoSwarmingTask creates a isolated.gen.json file using BUILD_REPO_ISOLATE, archives it, and triggers it's swarming task. The swarming task will run the build_repo worker script which will return a list of remote build directories.

func TriggerIsolateTelemetrySwarmingTask 

func TriggerIsolateTelemetrySwarmingTask(ctx context.Context, taskName, runID, chromiumHash, serviceAccountJSON string, patches []string, hardTimeout, ioTimeout time.Duration, local bool) (string, error)

TriggerIsolateTelemetrySwarmingTask creates a isolated.gen.json file using ISOLATE_TELEMETRY_ISOLATE, archives it, and triggers its swarming task. The swarming task will run the isolate_telemetry worker script which will return the isolate hash.

func TriggerSwarmingTask 

func TriggerSwarmingTask(ctx context.Context, pagesetType, taskPrefix, isolateName, runID, serviceAccountJSON string, hardTimeout, ioTimeout time.Duration, priority, maxPagesPerBot, numPages int, isolateExtraArgs map[string]string, runOnGCE, local bool, repeatValue int, isolateDeps []string) (int, error)

TriggerSwarmingTask returns the number of triggered tasks and an error (if any).

func ValidateSKPs 

func ValidateSKPs(ctx context.Context, pathToSkps, pathToPyFiles, pathToSkpinfo string) error

ValidateSKPs moves all root_dir/index/dir_name/*.skp into the root_dir/index and validates them. SKPs that fail validation are logged and deleted.

func WritePageset 

func WritePageset(filePath, userAgent, archiveFilePath, url string) error

Types

type CTBenchData 

type CTBenchData struct {
	*ingestcommon.BenchData
	RunID string `json:"runID"`
}

Extend ingestcommon.BenchData to include RunID.

type ClientConfig 

type ClientConfig struct {
	ClientID     string `json:"client_id"`
	ClientSecret string `json:"client_secret"`
}

type GcsUtil 

type GcsUtil struct {
	// contains filtered or unexported fields
}

func NewGcsUtil 

func NewGcsUtil(client *http.Client) (*GcsUtil, error)

NewGcsUtil initializes and returns a utility for CT interations with Google Storage. If client is nil then a client is created either from ClientSecretPath or with the default token source.

func (*GcsUtil) DeleteRemoteDir 

func (gs *GcsUtil) DeleteRemoteDir(gsDir string) error

func (*GcsUtil) DeleteRemoteDirLogErr 

func (gs *GcsUtil) DeleteRemoteDirLogErr(gsDir string)

DeleteRemoteDirLogErr wraps DeleteRemoteDir and logs an error if one is returned.

func (*GcsUtil) DownloadChromiumBuild 

func (gs *GcsUtil) DownloadChromiumBuild(chromiumBuild string) error

DownloadChromiumBuild downloads the specified Chromium build from Google Storage to a local dir.

func (*GcsUtil) DownloadRemoteFile 

func (gs *GcsUtil) DownloadRemoteFile(remotePath, localPath string) error

DownloadRemoteFile calls DownloadRemoteFileFromBucket with CT's default bucket.

func (*GcsUtil) DownloadRemoteFileFromBucket 

func (gs *GcsUtil) DownloadRemoteFileFromBucket(bucket, remotePath, localPath string) error

DownloadRemoteFileFromBucket downloads the specified remote path into the specified local file. This function has been tested to download very large files (~33GB). TODO(rmistry): Update all code that downloads remote files to use this or the DownloadRemoteFile method.

func (*GcsUtil) DownloadSwarmingArtifacts 

func (gs *GcsUtil) DownloadSwarmingArtifacts(localDir, remoteDirName, pagesetType string, startRange, num int) (map[string]int, error)

DownloadSwarmingArtifacts downloads the specified artifacts from Google Storage to a local dir. The Google storage directory is assumed to have numerical subdirs Eg: {1..1000}. This function downloads the contents of those directories into a local directory without the numerical subdirs. Returns the ranking/index of the downloaded artifact.

func (*GcsUtil) GetRemoteDirCount 

func (gs *GcsUtil) GetRemoteDirCount(gsDir string) (int, error)

GetRemoteDirCount returns the number of objects in the specified dir.

func (*GcsUtil) GetRemoteFileContents 

func (gs *GcsUtil) GetRemoteFileContents(filePath string) (io.ReadCloser, error)

Returns the response body of the specified GCS file from the default CT bucket. Client must close the response body when finished with it.

func (*GcsUtil) GetRemoteFileContentsFromBucket 

func (gs *GcsUtil) GetRemoteFileContentsFromBucket(bucket, filePath string) (io.ReadCloser, error)

Returns the response body of the specified GCS file. Client must close the response body when finished with it.

func (*GcsUtil) UploadDir 

func (gs *GcsUtil) UploadDir(localDir, gsDir string, cleanDir bool) error

UploadDir uploads the specified local dir into the specified Google Storage dir.

func (*GcsUtil) UploadFile 

func (gs *GcsUtil) UploadFile(fileName, localDir, gsDir string) error

UploadFile calls UploadFileToBucket with CT's default bucket.

func (*GcsUtil) UploadFileToBucket 

func (gs *GcsUtil) UploadFileToBucket(fileName, localDir, gsDir, bucket string) error

UploadFileToBucket uploads the specified file to the remote dir of the bucket in Google Storage. It also sets the appropriate ACLs on the uploaded file.

func (*GcsUtil) UploadSwarmingArtifacts 

func (gs *GcsUtil) UploadSwarmingArtifacts(dirName, pagesetType string) error

UploadSwarmingArtifact uploads the specified local artifacts to Google Storage.

type Installed 

type Installed struct {
	Installed ClientConfig `json:"installed"`
}

type Pageset 

type Pageset struct {
	UserAgent       string `json:"user_agent"`
	ArchiveDataFile string `json:"archive_data_file"`
	UrlsList        string `json:"urls_list"`
}

type PagesetTypeInfo 

type PagesetTypeInfo struct {
	NumPages                   int
	CSVSource                  string
	UserAgent                  string
	CaptureArchivesTimeoutSecs int
	CreatePagesetsTimeoutSecs  int
	CaptureSKPsTimeoutSecs     int
	PixelDiffTimeoutSecs       int
	RunChromiumPerfTimeoutSecs int
	Description                string
}

type PagesetVars 

type PagesetVars struct {
	// A comma separated list of URLs.
	UrlsList string `json:"urls_list"`
	// Will be either "mobile" or "desktop".
	UserAgent string `json:"user_agent"`
	// The location of the web page's WPR data file.
	ArchiveDataFile string `json:"archive_data_file"`
}

Contains the data included in CT pagesets.

func ReadPageset 

func ReadPageset(pagesetPath string) (PagesetVars, error)

type TimeoutTracker 

type TimeoutTracker struct {
	// contains filtered or unexported fields
}

func (*TimeoutTracker) Increment 

func (t *TimeoutTracker) Increment()

func (*TimeoutTracker) Read 

func (t *TimeoutTracker) Read() int

func (*TimeoutTracker) Reset 

func (t *TimeoutTracker) Reset()

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.