README

CQ BigQuery Schema

This includes the schema for rows in the CQ attempts table, as well as related scripts to create or update the schema.

Creating tables

All tables belong to a dataset, which has a name and description. Datasets can be created with the gcloud SDK bq command, which must be installed first. Datasets can be used for permission boundaries. So, to control access for different projects, the dataset used for all data directly from CQ can be put in its own dataset, whereas per-project views can be put in separate datasets.

Datasets can be created with command that look like:

bq --location=US mk --dataset --description "Attempts" commit-queue:raw

Schemas can be updated with bqschemaupdater. You should make sure you run an up-to-date bqschemaupdater:

go install go.chromium.org/luci/tools/cmd/bqschemaupdater

This tool takes a proto message and table and updates the schema based on the compiled proto message. To compile the proto message and update the schema:

go generate
bqschemaupdater -message bigquery.Attempt -table commit-queue.raw.attempts

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	Mode_name = map[int32]string{
		0: "MODE_UNSPECIFIED",
		1: "DRY_RUN",
		2: "FULL_RUN",
		3: "QUICK_DRY_RUN",
	}
	Mode_value = map[string]int32{
		"MODE_UNSPECIFIED": 0,
		"DRY_RUN":          1,
		"FULL_RUN":         2,
		"QUICK_DRY_RUN":    3,
	}
)

    Enum value maps for Mode.

    View Source
    var (
    	AttemptStatus_name = map[int32]string{
    		0: "ATTEMPT_STATUS_UNSPECIFIED",
    		1: "STARTED",
    		2: "SUCCESS",
    		3: "ABORTED",
    		4: "FAILURE",
    		5: "INFRA_FAILURE",
    	}
    	AttemptStatus_value = map[string]int32{
    		"ATTEMPT_STATUS_UNSPECIFIED": 0,
    		"STARTED":                    1,
    		"SUCCESS":                    2,
    		"ABORTED":                    3,
    		"FAILURE":                    4,
    		"INFRA_FAILURE":              5,
    	}
    )

      Enum value maps for AttemptStatus.

      View Source
      var (
      	AttemptSubstatus_name = map[int32]string{
      		0: "ATTEMPT_SUBSTATUS_UNSPECIFIED",
      		1: "NO_SUBSTATUS",
      		2: "FAILED_TRYJOBS",
      		3: "FAILED_LINT",
      		4: "UNAPPROVED",
      		5: "PERMISSION_DENIED",
      		6: "UNSATISFIED_DEPENDENCY",
      		7: "MANUAL_CANCEL",
      		8: "BUILDBUCKET_MISCONFIGURATION",
      	}
      	AttemptSubstatus_value = map[string]int32{
      		"ATTEMPT_SUBSTATUS_UNSPECIFIED": 0,
      		"NO_SUBSTATUS":                  1,
      		"FAILED_TRYJOBS":                2,
      		"FAILED_LINT":                   3,
      		"UNAPPROVED":                    4,
      		"PERMISSION_DENIED":             5,
      		"UNSATISFIED_DEPENDENCY":        6,
      		"MANUAL_CANCEL":                 7,
      		"BUILDBUCKET_MISCONFIGURATION":  8,
      	}
      )

        Enum value maps for AttemptSubstatus.

        View Source
        var (
        	GerritChange_SubmitStatus_name = map[int32]string{
        		0: "SUBMIT_STATUS_UNSPECIFIED",
        		1: "PENDING",
        		2: "UNKNOWN",
        		3: "FAILURE",
        		4: "SUCCESS",
        	}
        	GerritChange_SubmitStatus_value = map[string]int32{
        		"SUBMIT_STATUS_UNSPECIFIED": 0,
        		"PENDING":                   1,
        		"UNKNOWN":                   2,
        		"FAILURE":                   3,
        		"SUCCESS":                   4,
        	}
        )

          Enum value maps for GerritChange_SubmitStatus.

          View Source
          var (
          	Build_Origin_name = map[int32]string{
          		0: "ORIGIN_UNSPECIFIED",
          		1: "NOT_REUSABLE",
          		2: "NOT_REUSED",
          		3: "REUSED",
          	}
          	Build_Origin_value = map[string]int32{
          		"ORIGIN_UNSPECIFIED": 0,
          		"NOT_REUSABLE":       1,
          		"NOT_REUSED":         2,
          		"REUSED":             3,
          	}
          )

            Enum value maps for Build_Origin.

            View Source
            var File_go_chromium_org_luci_cv_api_bigquery_v1_attempt_proto protoreflect.FileDescriptor

            Functions

            This section is empty.

            Types

            type Attempt

            type Attempt struct {
            
            	// The opaque key unique to this Attempt.
            	Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
            	// The LUCI project that this Attempt belongs to.
            	LuciProject string `protobuf:"bytes,2,opt,name=luci_project,json=luciProject,proto3" json:"luci_project,omitempty"`
            	// The name of the config group that this Attempt belongs to.
            	ConfigGroup string `protobuf:"bytes,11,opt,name=config_group,json=configGroup,proto3" json:"config_group,omitempty"`
            	// An opaque key that is unique for a given set of Gerrit change patchsets.
            	// (or, equivalently, buildsets). The same cl_group_key will be used if
            	// another Attempt is made for the same set of changes at a different time.
            	ClGroupKey string `protobuf:"bytes,3,opt,name=cl_group_key,json=clGroupKey,proto3" json:"cl_group_key,omitempty"`
            	// Similar to cl_group_key, except the key will be the same when the
            	// earliest_equivalent_patchset values are the same, even if the patchset
            	// values are different.
            	//
            	// For example, when a new "trivial" patchset is uploaded, then the
            	// cl_group_key will change but the equivalent_cl_group_key will stay the
            	// same.
            	EquivalentClGroupKey string `protobuf:"bytes,4,opt,name=equivalent_cl_group_key,json=equivalentClGroupKey,proto3" json:"equivalent_cl_group_key,omitempty"`
            	// The time when the Attempt started (trigger time of the last CL triggered).
            	StartTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"`
            	// The time when the Attempt ended (released by CQ).
            	EndTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"`
            	// Gerrit changes, with specific patchsets, in this Attempt.
            	// There should be one or more.
            	GerritChanges []*GerritChange `protobuf:"bytes,7,rep,name=gerrit_changes,json=gerritChanges,proto3" json:"gerrit_changes,omitempty"`
            	// Relevant builds as of this Attempt's end time.
            	//
            	// While Attempt is processed, CQ may consider more builds than included here.
            	//
            	// For example, the following builds will be not be included:
            	//   * builds triggered before this Attempt started, considered temporarily by
            	//     CQ, but then ignored because they ultimately failed such that CQ had to
            	//     trigger new builds instead.
            	//   * successful builds which were fresh enough at the Attempt start time,
            	//     but which were ignored after they became too old for consideration such
            	//     that CQ had to trigger new builds instead.
            	//   * builds triggered as part of this Attempt, which were later removed from
            	//     project CQ config and hence were no longer required by CQ by Attempt
            	//     end time.
            	Builds []*Build `protobuf:"bytes,8,rep,name=builds,proto3" json:"builds,omitempty"`
            	// Final status of the Attempt.
            	Status AttemptStatus `protobuf:"varint,9,opt,name=status,proto3,enum=bigquery.AttemptStatus" json:"status,omitempty"`
            	// A more fine-grained status the explains more details about the status.
            	Substatus AttemptSubstatus `protobuf:"varint,10,opt,name=substatus,proto3,enum=bigquery.AttemptSubstatus" json:"substatus,omitempty"`
            	// Whether or not the required builds for this attempt include additional
            	// "opted-in" builders by the user via the `Cq-Include-Trybots` footer.
            	HasCustomRequirement bool `protobuf:"varint,12,opt,name=has_custom_requirement,json=hasCustomRequirement,proto3" json:"has_custom_requirement,omitempty"`
            	// contains filtered or unexported fields
            }

              Attempt includes the state of one CQ attempt.

              An attempt involves doing checks for one or more CLs that could potentially be submitted together.

              Next ID: 13.

              func (*Attempt) Descriptor

              func (*Attempt) Descriptor() ([]byte, []int)

                Deprecated: Use Attempt.ProtoReflect.Descriptor instead.

                func (*Attempt) GetBuilds

                func (x *Attempt) GetBuilds() []*Build

                func (*Attempt) GetClGroupKey

                func (x *Attempt) GetClGroupKey() string

                func (*Attempt) GetConfigGroup

                func (x *Attempt) GetConfigGroup() string

                func (*Attempt) GetEndTime

                func (x *Attempt) GetEndTime() *timestamppb.Timestamp

                func (*Attempt) GetEquivalentClGroupKey

                func (x *Attempt) GetEquivalentClGroupKey() string

                func (*Attempt) GetGerritChanges

                func (x *Attempt) GetGerritChanges() []*GerritChange

                func (*Attempt) GetHasCustomRequirement

                func (x *Attempt) GetHasCustomRequirement() bool

                func (*Attempt) GetKey

                func (x *Attempt) GetKey() string

                func (*Attempt) GetLuciProject

                func (x *Attempt) GetLuciProject() string

                func (*Attempt) GetStartTime

                func (x *Attempt) GetStartTime() *timestamppb.Timestamp

                func (*Attempt) GetStatus

                func (x *Attempt) GetStatus() AttemptStatus

                func (*Attempt) GetSubstatus

                func (x *Attempt) GetSubstatus() AttemptSubstatus

                func (*Attempt) ProtoMessage

                func (*Attempt) ProtoMessage()

                func (*Attempt) ProtoReflect

                func (x *Attempt) ProtoReflect() protoreflect.Message

                func (*Attempt) Reset

                func (x *Attempt) Reset()

                func (*Attempt) String

                func (x *Attempt) String() string

                type AttemptStatus

                type AttemptStatus int32
                const (
                	// Default, never set.
                	AttemptStatus_ATTEMPT_STATUS_UNSPECIFIED AttemptStatus = 0
                	// Started but not completed. Used by CQ API, TBD.
                	AttemptStatus_STARTED AttemptStatus = 1
                	// Ready to submit, all checks passed.
                	AttemptStatus_SUCCESS AttemptStatus = 2
                	// Attempt stopped before completion, due to some external event and not
                	// a failure of the CLs to pass all tests. For example, this may happen
                	// when a new patchset is uploaded, a CL is deleted, etc.
                	AttemptStatus_ABORTED AttemptStatus = 3
                	// Completed and failed some check. This may happen when a build failed,
                	// footer syntax was incorrect, or CL was not approved.
                	AttemptStatus_FAILURE AttemptStatus = 4
                	// Failure in CQ itself caused the Attempt to be dropped.
                	AttemptStatus_INFRA_FAILURE AttemptStatus = 5
                )

                func (AttemptStatus) Descriptor

                func (AttemptStatus) Enum

                func (x AttemptStatus) Enum() *AttemptStatus

                func (AttemptStatus) EnumDescriptor

                func (AttemptStatus) EnumDescriptor() ([]byte, []int)

                  Deprecated: Use AttemptStatus.Descriptor instead.

                  func (AttemptStatus) Number

                  func (AttemptStatus) String

                  func (x AttemptStatus) String() string

                  func (AttemptStatus) Type

                  type AttemptSubstatus

                  type AttemptSubstatus int32
                  const (
                  	// Default, never set.
                  	AttemptSubstatus_ATTEMPT_SUBSTATUS_UNSPECIFIED AttemptSubstatus = 0
                  	// There is no more detailed status set.
                  	AttemptSubstatus_NO_SUBSTATUS AttemptSubstatus = 1
                  	// Failed at least one critical tryjob.
                  	AttemptSubstatus_FAILED_TRYJOBS AttemptSubstatus = 2
                  	// Failed an initial quick check of CL and CL description state.
                  	AttemptSubstatus_FAILED_LINT AttemptSubstatus = 3
                  	// A CL didn't get sufficient approval for submitting via CQ.
                  	AttemptSubstatus_UNAPPROVED AttemptSubstatus = 4
                  	// A CQ triggerer doesn't have permission to trigger CQ.
                  	AttemptSubstatus_PERMISSION_DENIED AttemptSubstatus = 5
                  	// There was a problem with a dependency CL, e.g. some dependencies
                  	// were not submitted or not grouped together in this attempt.
                  	AttemptSubstatus_UNSATISFIED_DEPENDENCY AttemptSubstatus = 6
                  	// Aborted because of a manual cancelation.
                  	AttemptSubstatus_MANUAL_CANCEL AttemptSubstatus = 7
                  	// A request to buildbucket failed because CQ didn't have permission to
                  	// trigger builds.
                  	AttemptSubstatus_BUILDBUCKET_MISCONFIGURATION AttemptSubstatus = 8
                  )

                  func (AttemptSubstatus) Descriptor

                  func (AttemptSubstatus) Enum

                  func (AttemptSubstatus) EnumDescriptor

                  func (AttemptSubstatus) EnumDescriptor() ([]byte, []int)

                    Deprecated: Use AttemptSubstatus.Descriptor instead.

                    func (AttemptSubstatus) Number

                    func (AttemptSubstatus) String

                    func (x AttemptSubstatus) String() string

                    func (AttemptSubstatus) Type

                    type Build

                    type Build struct {
                    
                    	// Buildbucket build ID, unique per Buildbucket instance.
                    	Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
                    	// Buildbucket host, e.g. "cr-buildbucket.appspot.com".
                    	Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"`
                    	// Information about whether this build was triggered previously and reused,
                    	// or triggered because there was no reusable build, or because builds by
                    	// this builder are all not reusable.
                    	Origin Build_Origin `protobuf:"varint,3,opt,name=origin,proto3,enum=bigquery.Build_Origin" json:"origin,omitempty"`
                    	// Whether the CQ must wait for this build to pass in order for the CLs to be
                    	// considered ready to submit. True means this builder must pass, false means
                    	// this builder is "optional", and so this build should not be used to assess
                    	// the correctness of the CLs in the Attempt. For example, builds added
                    	// because of the Cq-Include-Trybots footer are still critical; experimental
                    	// builders are not.
                    	//
                    	// Tip: join this with the Buildbucket BigQuery table to figure out which
                    	// builder this build belongs to.
                    	Critical bool `protobuf:"varint,4,opt,name=critical,proto3" json:"critical,omitempty"`
                    	// contains filtered or unexported fields
                    }

                      Build represents one tryjob Buildbucket build.

                      See also: Build in buildbucket/proto/build.proto.

                      func (*Build) Descriptor

                      func (*Build) Descriptor() ([]byte, []int)

                        Deprecated: Use Build.ProtoReflect.Descriptor instead.

                        func (*Build) GetCritical

                        func (x *Build) GetCritical() bool

                        func (*Build) GetHost

                        func (x *Build) GetHost() string

                        func (*Build) GetId

                        func (x *Build) GetId() int64

                        func (*Build) GetOrigin

                        func (x *Build) GetOrigin() Build_Origin

                        func (*Build) ProtoMessage

                        func (*Build) ProtoMessage()

                        func (*Build) ProtoReflect

                        func (x *Build) ProtoReflect() protoreflect.Message

                        func (*Build) Reset

                        func (x *Build) Reset()

                        func (*Build) String

                        func (x *Build) String() string

                        type Build_Origin

                        type Build_Origin int32
                        const (
                        	// Default. Never set.
                        	Build_ORIGIN_UNSPECIFIED Build_Origin = 0
                        	// Build was triggered as part of this attempt
                        	// because reuse was disabled for its builder.
                        	Build_NOT_REUSABLE Build_Origin = 1
                        	// Build was triggered as part of this attempt,
                        	// but if there was an already existing build it would have been reused.
                        	Build_NOT_REUSED Build_Origin = 2
                        	// Build was reused.
                        	Build_REUSED Build_Origin = 3
                        )

                        func (Build_Origin) Descriptor

                        func (Build_Origin) Enum

                        func (x Build_Origin) Enum() *Build_Origin

                        func (Build_Origin) EnumDescriptor

                        func (Build_Origin) EnumDescriptor() ([]byte, []int)

                          Deprecated: Use Build_Origin.Descriptor instead.

                          func (Build_Origin) Number

                          func (Build_Origin) String

                          func (x Build_Origin) String() string

                          func (Build_Origin) Type

                          type GerritChange

                          type GerritChange struct {
                          
                          	// Gerrit hostname, e.g. "chromium-review.googlesource.com".
                          	Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"`
                          	// Gerrit project, e.g. "chromium/src".
                          	Project string `protobuf:"bytes,2,opt,name=project,proto3" json:"project,omitempty"`
                          	// Change number, e.g. 12345.
                          	Change int64 `protobuf:"varint,3,opt,name=change,proto3" json:"change,omitempty"`
                          	// Patch set number, e.g. 1.
                          	Patchset int64 `protobuf:"varint,4,opt,name=patchset,proto3" json:"patchset,omitempty"`
                          	// The earliest patchset of the CL that is considered equivalent to the
                          	// patchset above.
                          	EarliestEquivalentPatchset int64 `` /* 142-byte string literal not displayed */
                          	// The time that the CQ was triggered for this CL in this Attempt.
                          	TriggerTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=trigger_time,json=triggerTime,proto3" json:"trigger_time,omitempty"`
                          	// CQ Mode for this CL, e.g. dry run or full run.
                          	Mode Mode `protobuf:"varint,7,opt,name=mode,proto3,enum=bigquery.Mode" json:"mode,omitempty"`
                          	// Whether CQ tried to submit this change and the result of the operation.
                          	SubmitStatus GerritChange_SubmitStatus `` /* 138-byte string literal not displayed */
                          	// contains filtered or unexported fields
                          }

                            GerritChange represents one revision (patchset) of one Gerrit change in an Attempt.

                            See also: GerritChange in buildbucket/proto/common.proto.

                            func (*GerritChange) Descriptor

                            func (*GerritChange) Descriptor() ([]byte, []int)

                              Deprecated: Use GerritChange.ProtoReflect.Descriptor instead.

                              func (*GerritChange) GetChange

                              func (x *GerritChange) GetChange() int64

                              func (*GerritChange) GetEarliestEquivalentPatchset

                              func (x *GerritChange) GetEarliestEquivalentPatchset() int64

                              func (*GerritChange) GetHost

                              func (x *GerritChange) GetHost() string

                              func (*GerritChange) GetMode

                              func (x *GerritChange) GetMode() Mode

                              func (*GerritChange) GetPatchset

                              func (x *GerritChange) GetPatchset() int64

                              func (*GerritChange) GetProject

                              func (x *GerritChange) GetProject() string

                              func (*GerritChange) GetSubmitStatus

                              func (x *GerritChange) GetSubmitStatus() GerritChange_SubmitStatus

                              func (*GerritChange) GetTriggerTime

                              func (x *GerritChange) GetTriggerTime() *timestamppb.Timestamp

                              func (*GerritChange) ProtoMessage

                              func (*GerritChange) ProtoMessage()

                              func (*GerritChange) ProtoReflect

                              func (x *GerritChange) ProtoReflect() protoreflect.Message

                              func (*GerritChange) Reset

                              func (x *GerritChange) Reset()

                              func (*GerritChange) String

                              func (x *GerritChange) String() string

                              type GerritChange_SubmitStatus

                              type GerritChange_SubmitStatus int32
                              const (
                              	// Default. Never set.
                              	GerritChange_SUBMIT_STATUS_UNSPECIFIED GerritChange_SubmitStatus = 0
                              	// CQ didn't try submitting this CL.
                              	//
                              	// Includes a case where CQ tried submitting the CL, but submission failed
                              	// due to transient error leaving CL as is, and CQ didn't try again.
                              	GerritChange_PENDING GerritChange_SubmitStatus = 1
                              	// CQ tried to submit, but got presumably transient errors and couldn't
                              	// ascertain whether submission was successful.
                              	//
                              	// It's possible that change was actually submitted, but CQ didn't receive
                              	// a confirmation from Gerrit and follow up checks of the change status
                              	// failed, too.
                              	GerritChange_UNKNOWN GerritChange_SubmitStatus = 2
                              	// CQ tried to submit, but Gerrit rejected the submission because this
                              	// Change can't be submitted.
                              	// Typically, this is because a rebase conflict needs to be resolved,
                              	// or rarely because the change needs some kind of approval.
                              	GerritChange_FAILURE GerritChange_SubmitStatus = 3
                              	// CQ submitted this change (aka "merged" in Gerrit jargon).
                              	//
                              	// Submission of Gerrit CLs in an Attempt is not an atomic operation,
                              	// so it's possible that only some of the GerritChanges are submitted.
                              	GerritChange_SUCCESS GerritChange_SubmitStatus = 4
                              )

                              func (GerritChange_SubmitStatus) Descriptor

                              func (GerritChange_SubmitStatus) Enum

                              func (GerritChange_SubmitStatus) EnumDescriptor

                              func (GerritChange_SubmitStatus) EnumDescriptor() ([]byte, []int)

                                Deprecated: Use GerritChange_SubmitStatus.Descriptor instead.

                                func (GerritChange_SubmitStatus) Number

                                func (GerritChange_SubmitStatus) String

                                func (x GerritChange_SubmitStatus) String() string

                                func (GerritChange_SubmitStatus) Type

                                type Mode

                                type Mode int32
                                const (
                                	// Default, never set.
                                	Mode_MODE_UNSPECIFIED Mode = 0
                                	// Run all tests but do not submit.
                                	Mode_DRY_RUN Mode = 1
                                	// Run all tests and potentially submit.
                                	Mode_FULL_RUN Mode = 2
                                	// Run some tests using RTS, and do not submit.
                                	Mode_QUICK_DRY_RUN Mode = 3
                                )

                                func (Mode) Descriptor

                                func (Mode) Descriptor() protoreflect.EnumDescriptor

                                func (Mode) Enum

                                func (x Mode) Enum() *Mode

                                func (Mode) EnumDescriptor

                                func (Mode) EnumDescriptor() ([]byte, []int)

                                  Deprecated: Use Mode.Descriptor instead.

                                  func (Mode) Number

                                  func (x Mode) Number() protoreflect.EnumNumber

                                  func (Mode) String

                                  func (x Mode) String() string

                                  func (Mode) Type

                                  func (Mode) Type() protoreflect.EnumType