google

package
v0.0.0-...-8515a6d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 12, 2018 License: MIT Imports: 20 Imported by: 0

README

google cloud storage store

Cloudstorage abstraction package for gcs.

# the CloudStorage GCS JWT key is an env with full jwt token json encoded.
export CS_GCS_JWTKEY="{\"project_id\": \"lio-testing\", \"private_key_id\": \"

Example


// example with  CS_GCS_JWTKEY env var
conf := &cloudstorage.Config{
	Type:       google.StoreType,
	AuthMethod: google.AuthJWTKeySource,
	Project:    "my-google-project",
	Bucket:     "integration-tests-nl",
	TmpDir:     "/tmp/localcache/google",
}

// OR read from machine oauth locations
conf := &cloudstorage.Config{
	Type:       google.StoreType,
	AuthMethod: google.AuthGCEDefaultOAuthToken,
	Project:    "my-google-project",
	Bucket:     "integration-tests-nl",
	TmpDir:     "/tmp/localcache/google",
}

// OR metadata api if on google cloud
conf := &cloudstorage.Config{
	Type:       google.StoreType,
	AuthMethod: google.AuthGCEMetaKeySource,
	Project:    "my-google-project",
	Bucket:     "integration-tests-nl",
	TmpDir:     "/tmp/localcache/google",
}

// create store
store, err := cloudstorage.NewStore(conf)
if err != nil {
    return err
}

Documentation

Index

Constants

View Source 
const (

	// AuthJWTKeySource is for a complete string representing json of JWT
	AuthJWTKeySource cloudstorage.AuthMethod = "LyticsJWTkey"
	// AuthGoogleJWTKeySource is a string representing path to a file of JWT
	AuthGoogleJWTKeySource cloudstorage.AuthMethod = "GoogleJWTFile"
	// AuthGCEMetaKeySource is flag saying to use gcemetadata
	AuthGCEMetaKeySource cloudstorage.AuthMethod = "gcemetadata"
	// AuthGCEDefaultOAuthToken means use local auth where it (google client)
	// checks variety of locations for local auth tokens.
	AuthGCEDefaultOAuthToken cloudstorage.AuthMethod = "gcedefaulttoken"
)
View Source 
const StoreType = "gcs"

StoreType = "gcs"

Variables

View Source 
var (
	// GCSRetries number of times to retry for GCS.
	GCSRetries int = 55
)

Functions

This section is empty.

Types

type APIStore 

type APIStore struct {
	// contains filtered or unexported fields
}

APIStore a google api store

func NewAPIStore 

func NewAPIStore(conf *cloudstorage.Config) (*APIStore, error)

NewAPIStore create api store.

func (*APIStore) AddOwner 

func (c *APIStore) AddOwner(bucket, object, entity string) error

AddOwner adds entity as a owner of the object

func (*APIStore) AddReader 

func (c *APIStore) AddReader(bucket, object, entity string) error

AddReader adds enitty as a reader of the object

func (*APIStore) BucketExists 

func (c *APIStore) BucketExists(name string) bool

BucketExists checks for the bucket name

func (*APIStore) CreateBucket 

func (c *APIStore) CreateBucket(name string) error

CreateBucket creates a new bucket in GCS

type GcsFS 

type GcsFS struct {
	PageSize int
	Id       string
	// contains filtered or unexported fields
}

GcsFS Simple wrapper for accessing smaller GCS files, it doesn't currently implement a Reader/Writer interface so not useful for stream reading of large files yet.

func NewGCSStore 

func NewGCSStore(gcs *storage.Client, bucket, cachepath string, pagesize int) (*GcsFS, error)

NewGCSStore Create Google Cloud Storage Store.

func (*GcsFS) Client 

func (g *GcsFS) Client() interface{}

Client gets access to the underlying google cloud storage client.

func (*GcsFS) Copy 

func (g *GcsFS) Copy(ctx context.Context, src, des cloudstorage.Object) error

Copy from src to destination

func (*GcsFS) Delete 

func (g *GcsFS) Delete(ctx context.Context, obj string) error

Delete requested object path string.

func (*GcsFS) Folders 

func (g *GcsFS) Folders(ctx context.Context, csq cloudstorage.Query) ([]string, error)

Folders get folders list.

func (*GcsFS) Get 

func (g *GcsFS) Get(ctx context.Context, objectpath string) (cloudstorage.Object, error)

Get Gets a single File Object

func (*GcsFS) List 

func (g *GcsFS) List(ctx context.Context, csq cloudstorage.Query) (*cloudstorage.ObjectsResponse, error)

Objects returns an iterator over the objects in the google bucket that match the Query q. If q is nil, no filtering is done.

func (*GcsFS) Move 

func (g *GcsFS) Move(ctx context.Context, src, des cloudstorage.Object) error

Move which is a Copy & Delete

func (*GcsFS) NewObject 

func (g *GcsFS) NewObject(objectname string) (cloudstorage.Object, error)

NewObject of Type GCS.

func (*GcsFS) NewReader 

func (g *GcsFS) NewReader(o string) (io.ReadCloser, error)

NewReader create GCS file reader.

func (*GcsFS) NewReaderWithContext 

func (g *GcsFS) NewReaderWithContext(ctx context.Context, o string) (io.ReadCloser, error)

NewReaderWithContext create new GCS File reader with context.

func (*GcsFS) NewWriter 

func (g *GcsFS) NewWriter(o string, metadata map[string]string) (io.WriteCloser, error)

NewWriter create GCS Object Writer.

func (*GcsFS) NewWriterWithContext 

func (g *GcsFS) NewWriterWithContext(ctx context.Context, o string, metadata map[string]string) (io.WriteCloser, error)

NewWriterWithContext create writer with provided context and metadata.

func (*GcsFS) Objects 

func (g *GcsFS) Objects(ctx context.Context, csq cloudstorage.Query) (cloudstorage.ObjectIterator, error)

Objects returns an iterator over the objects in the google bucket that match the Query q. If q is nil, no filtering is done.

func (*GcsFS) String 

func (g *GcsFS) String() string

String function to provide gs://..../file path

func (*GcsFS) Type 

func (g *GcsFS) Type() string

Type of store = "gcs"

type GoogleOAuthClient 

type GoogleOAuthClient interface {
	Client() *http.Client
}

GoogleOAuthClient An interface so we can return any of the 3 Google transporter wrapper as a single interface.

func BuildDefaultGoogleTransporter 

func BuildDefaultGoogleTransporter(scope ...string) (GoogleOAuthClient, error)

BuildDefaultGoogleTransporter builds a transpoter that wraps the google DefaultClient: 

Ref https://github.com/golang/oauth2/blob/master/google/default.go#L33

DefaultClient returns an HTTP Client that uses the DefaultTokenSource to obtain authentication credentials 

Ref : https://github.com/golang/oauth2/blob/master/google/default.go#L41

DefaultTokenSource is a token source that uses "Application Default Credentials".

It looks for credentials in the following places, preferring the first location found:

  1. A JSON file whose path is specified by the GOOGLE_APPLICATION_CREDENTIALS environment variable.
  2. A JSON file in a location known to the gcloud command-line tool. On other systems, $HOME/.config/gcloud/credentials.
  3. On Google App Engine it uses the appengine.AccessToken function.
  4. On Google Compute Engine, it fetches credentials from the metadata server. (In this final case any provided scopes are ignored.)

For more details, see: https://developers.google.com/accounts/docs/application-default-credentials

Samples of possible scopes: Google Cloud Storage : https://github.com/GoogleCloudPlatform/gcloud-golang/blob/69098363d921fa3cf80f930468a41a33edd9ccb9/storage/storage.go#L51 BigQuery : https://github.com/GoogleCloudPlatform/gcloud-golang/blob/522a8ceb4bb83c2def27baccf31d646bce11a4b2/bigquery/bigquery.go#L52

func BuildGCEMetadatTransporter 

func BuildGCEMetadatTransporter(serviceAccount string) (GoogleOAuthClient, error)

The account may be empty or the string "default" to use the instance's main account.

func BuildGoogleFileJWTTransporter 

func BuildGoogleFileJWTTransporter(keyPath string, scope string) (GoogleOAuthClient, error)

BuildGoogleFileJWTTransporter Build a Google Storage Client from a path to a json file that has JWT.

func BuildGoogleJWTTransporter 

func BuildGoogleJWTTransporter(jwtConf *cloudstorage.JwtConf) (GoogleOAuthClient, error)

BuildGoogleJWTTransporter create a GoogleOAuthClient from jwt config.

func NewGoogleClient 

func NewGoogleClient(conf *cloudstorage.Config) (client GoogleOAuthClient, err error)

NewGoogleClient create new Google Storage Client.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.