Documentation

Overview

    Package remoteexec provides proxy to remoteexec server.

    TODO: goma client should use credential described in https://cloud.google.com/remote-build-execution/docs/getting-started

    Index

    Constants

    This section is empty.

    Variables

    View Source
    var (
    	DefaultViews = []*view.View{
    		{
    			Description: `Number of current running exec operations`,
    			Measure:     numRunningOperations,
    			Aggregation: view.Sum(),
    		},
    		{
    			Description: "Number of requests per wrapper types",
    			TagKeys: []tag.Key{
    				wrapperTypeKey,
    			},
    			Measure:     wrapperCount,
    			Aggregation: view.Count(),
    		},
    		{
    			Description: "Size to allocate buffer for input files",
    			TagKeys: []tag.Key{
    				allocStatusKey,
    			},
    			Measure:     inputBufferAllocSize,
    			Aggregation: view.Sum(),
    		},
    		{
    			Description: "Time in inventory check",
    			Measure:     execInventoryTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in input tree construction",
    			Measure:     execInputTreeTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in setup",
    			Measure:     execSetupTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time to check cache",
    			Measure:     execCheckCacheTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time to check missing",
    			Measure:     execCheckMissingTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time to upload blobs",
    			Measure:     execUploadBlobsTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time to execute",
    			Measure:     execExecuteTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in response",
    			Measure:     execResponseTime,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in RBE queue",
    			Measure:     rbeQueueTime,
    			TagKeys:     rbeTagKeys,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in RBE worker",
    			Measure:     rbeWorkerTime,
    			TagKeys:     rbeTagKeys,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in RBE input",
    			Measure:     rbeInputTime,
    			TagKeys:     rbeTagKeys,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in RBE exec",
    			Measure:     rbeExecTime,
    			TagKeys:     rbeTagKeys,
    			Aggregation: defaultLatencyDistribution,
    		},
    		{
    			Description: "Time in RBE output",
    			Measure:     rbeOutputTime,
    			TagKeys:     rbeTagKeys,
    			Aggregation: defaultLatencyDistribution,
    		},
    	}
    )

    Functions

    func ExecuteAndWait

    func ExecuteAndWait(ctx context.Context, c Client, req *rpb.ExecuteRequest, opts ...grpc.CallOption) (string, *rpb.ExecuteResponse, error)

      ExecuteAndWait executes and action remotely and wait its response. it returns operation name, response and error.

      Types

      type Adapter

      type Adapter struct {
      	// InstancePrefix is the prefix (dirname) of the full RBE instance name.
      	// e.g. If instance name == "projects/$PROJECT/instances/default_instance",
      	// then InstancePrefix is "projects/$PROJECT/instances"
      	InstancePrefix string
      
      	Inventory   exec.Inventory
      	ExecTimeout time.Duration
      
      	// Client is remoteexec API client.
      	Client         Client
      	InsecureClient bool
      
      	// GomaFile handles output files from remoteexec's cas to goma's FileBlob.
      	GomaFile fpb.FileServiceClient
      
      	// key: goma file hash.
      	DigestCache DigestCache
      
      	// CmdStorage is a storage for command files.
      	CmdStorage CmdStorage
      
      	// Tool details put in request metadata.
      	ToolDetails *rpb.ToolDetails
      
      	// FileLookupSema specifies concurrency to look up file
      	// contents from file-cache-server to be converted to CAS.
      	FileLookupSema chan struct{}
      
      	// CASBlobLookupSema specifies concurrency to look up file blobs in in cas.lookupBlobsInStore(),
      	// which calls Store.Get().
      	CASBlobLookupSema chan struct{}
      
      	// OutputFileSema specifies concurrency to download files from CAS and store in
      	// file server in gomaOutput.toFileBlob().
      	OutputFileSema chan struct{}
      
      	// Ratio to enable hardening.
      	HardeningRatio float64
      	// contains filtered or unexported fields
      }

        Adapter is an adapter from goma API to remoteexec API.

        func (*Adapter) DefaultInstance

        func (f *Adapter) DefaultInstance() string

        func (*Adapter) Exec

        func (f *Adapter) Exec(ctx context.Context, req *gomapb.ExecReq) (resp *gomapb.ExecResp, err error)

          Exec handles goma Exec requests with remoteexec backend.

          1. compute input tree and Action.
          1.1 construct input tree from req.
          1.2. construct Action message from req.
          2. checks the ActionCache using GetActionResult. if hit, go to 7.
          3. queries the ContentAddressableStorage using FindMissingBlobs
          4. uploads any missing blobs to the ContentAddressableStorage
             using bytestream.Write and BatchUpdateBlobs.
          5. executes the action using Execute.
          6. awaits completion of the action using the longrunning.Operations.
          7. looks a the ActionResult
          8. If the action is successful, uses bytestream.Read to download any outputs
             it does not already have;
             embed it in response, or will serve it by LookupFile later
          9. job is complete
          9.1  convert ExecResp from ExecuteResponse.
               for small outputs, embed in resp. otherwise use FILE_META.
          

          type ByteStream

          type ByteStream struct {
          	// Adapter provides an interface to the RBE API.
          	Adapter *Adapter
          
          	// The name of the RBE instance, e.g. "projects/$PROJECT/instances/default_instance"
          	InstanceName string
          }

            ByteStream is a proxy that reads/writes data to/from a server, as defined in googleapis/bytestream. In the context of the exec server, it accesses a resource on the RBE server, specifically a resource under an RBE instance.

            func (*ByteStream) QueryWriteStatus

              Write proxies bytestream QueryWriteStatus call.

              func (*ByteStream) Read

                Read proxies bytestream Read stream.

                func (*ByteStream) Write

                  Write proxies bytestream Write stream.

                  type Client

                  type Client struct {
                  	*grpc.ClientConn
                  	CallOptions []grpc.CallOption
                  	Retry       rpc.Retry
                  }

                    Client is a remoteexec API client to ClientConn. CallOptions will be added when calling RPC.

                    prcred, _ := oauth.NewApplicationDefault(ctx,
                       "https://www.googleapis.com/auth/cloud-build-service")
                    conn, _ := grpc.DialContext(ctx, target,
                      grpc.WithPerRPCCredentials(prcred),
                      grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
                    client := &remoteexec.Client{conn}
                    

                    func (Client) BatchReadBlobs

                    func (c Client) BatchReadBlobs(ctx context.Context, req *rpb.BatchReadBlobsRequest, opts ...grpc.CallOption) (*rpb.BatchReadBlobsResponse, error)

                      BatchReadBlobs downloads many blobs at once.

                      func (Client) BatchUpdateBlobs

                        BatchUpdateBlobs uploads many blobs at once.

                        func (Client) ByteStream

                        func (c Client) ByteStream() bpb.ByteStreamClient

                          ByteStream returns byte stream client. https://godoc.org/google.golang.org/genproto/googleapis/bytestream#ByteStreamClient

                          func (Client) Capabilities

                          func (c Client) Capabilities() rpb.CapabilitiesClient

                            Capabilities returns capabilities client.

                            func (Client) Execute

                              Execute executes an action remotely.

                              func (Client) FindMissingBlobs

                                FindMissingBlobs determines if blobs are present in the CAS.

                                func (Client) GetActionResult

                                func (c Client) GetActionResult(ctx context.Context, req *rpb.GetActionResultRequest, opts ...grpc.CallOption) (*rpb.ActionResult, error)

                                  GetActionResult retrieves a cached execution result.

                                  func (Client) GetCapabilities

                                  func (c Client) GetCapabilities(ctx context.Context, req *rpb.GetCapabilitiesRequest, opts ...grpc.CallOption) (*rpb.ServerCapabilities, error)

                                    GetCapabilities returns the server capabilities configuration.

                                    func (Client) GetTree

                                      GetTree fetches the entire directory tree rooted at a node.

                                      func (Client) QueryWriteStatus

                                        QueryWriteStatus is used to find the committed_size for a resource that is being written, which can be then be used as the write_offset for the next Write call.

                                        func (Client) Read

                                          Read is used to retrieve the contents of a resource as a sequence of bytes.

                                          func (Client) UpdateActionResult

                                          func (c Client) UpdateActionResult(ctx context.Context, req *rpb.UpdateActionResultRequest, opts ...grpc.CallOption) (*rpb.ActionResult, error)

                                            UpdateActionResult uploads a new execution result.

                                            func (Client) WaitExecution

                                              WaitExecution waits for an execution operation to complete.

                                              func (Client) Write

                                                Write is used to send the contents of a resource as a sequence of bytes.

                                                type CmdStorage

                                                type CmdStorage interface {
                                                	Open(ctx context.Context, hash string) (io.ReadCloser, error)
                                                }

                                                  CmdStorage is an interface to retrieve cmd file contents.

                                                  type DigestCache

                                                  type DigestCache interface {
                                                  	Get(context.Context, string, digest.Source) (digest.Data, error)
                                                  }

                                                    DigetCache caches digest for goma file hash.

                                                    Directories

                                                    Path Synopsis
                                                    Package cas manages content addressable storage.
                                                    Package cas manages content addressable storage.
                                                    Package datasource provides data source from local file, bytes etc.
                                                    Package datasource provides data source from local file, bytes etc.
                                                    Package digest handles content digest for remote executon API, https://github.com/bazelbuild/remote-apis/blob/c1c1ad2c97ed18943adb55f06657440daa60d833/build/bazel/remote/execution/v2/remote_execution.proto#L633
                                                    Package digest handles content digest for remote executon API, https://github.com/bazelbuild/remote-apis/blob/c1c1ad2c97ed18943adb55f06657440daa60d833/build/bazel/remote/execution/v2/remote_execution.proto#L633
                                                    Package merkletree operates on a merkle tree for remote execution API, https://github.com/bazelbuild/remote-apis/blob/c1c1ad2c97ed18943adb55f06657440daa60d833/build/bazel/remote/execution/v2/remote_execution.proto#L838 see https://en.Wikipedia.org/wiki/Merkle_tree
                                                    Package merkletree operates on a merkle tree for remote execution API, https://github.com/bazelbuild/remote-apis/blob/c1c1ad2c97ed18943adb55f06657440daa60d833/build/bazel/remote/execution/v2/remote_execution.proto#L838 see https://en.Wikipedia.org/wiki/Merkle_tree