README
¶
gs - Google Storage for Abstract File Storage
Usage
import (
"context"
"fmt"
"github.com/viant/afs"
_ "github.com/viant/afsc/gs"
"io/io"
"log"
)
func main() {
service := afs.New()
ctx := context.Background()
objects, err := service.List(ctx, "gs://myBucket/folder")
if err != nil {
log.Fatal(err)
}
for _, object := range objects {
fmt.Printf("%v %v\n", object.Name(), object.URL())
if object.IsDir() {
continue
}
reader, err := service.Download(ctx, object)
if err != nil {
log.Fatal(err)
}
defer reader.Close()
data, err := io.ReadAll(reader)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", data)
}
err = service.Copy(ctx, "gs://myBucket/folder", "/tmp")
if err != nil {
log.Fatal(err)
}
}
Auth
- Default Google Auth
When this library is used within GCP then owner service account is used to authenticate.
Otherwise use case use GOOGLE_APPLICATION_CREDENTIALS
- JSON Web Token (JWT)
To use JWT auth, provide a type that implements the following interface, you can also use gs.NewJwtConfig
type JWTProvider interface {
JWTConfig(scopes ...string) (config *jwt.Config, projectID string, err error)
}
Example:
secretPath := path.Join(os.Getenv("HOME"), ".secret", TestCredenitlas)
jwtConfig, err := gs.NewJwtConfig(option.NewLocation(secretPath))
if err != nil {
log.Fatal(err)
}
service := afs.New()
reader, err := service.DownloadWithURL(ctx, "gs://my-bucket/myfolder/asset.txt", jwtConfig)
data, err := io.ReadAll(reader)
if err != nil {
log.Fatal(err)
}
fmt.Printf("data: %s\n", data)
Options
secretPath := path.Join(os.Getenv("HOME"), ".secret", "gcp-e2e.json")
jwtConfig, err := gs.NewJwtConfig(option.NewLocation(secretPath))
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
JSON, err := json.Marshal(jwtConfig)
//import goption "google.golang.org/api/option"
jsonAuth := goption.WithCredentialsJSON(JSON)
service := afs.New()
reader, err := service.DownloadWithURL(ctx, "gs://my-bucket/myfolder/asset.txt", gs.NewClientOptions(jsonAuth), gs.NewProject("myproject"))
data, err := io.ReadAll(reader)
if err != nil {
log.Fatal(err)
}
fmt.Printf("data: %s\n", data)
- Custom key encryption
customKey, err := option.NewAES256Key([]byte("secret-key-that-is-32-bytes-long"))
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
service := afs.New()
err = service.Upload(ctx, "gs://mybucket/folder/secret1.txt", 0644, strings.NewReader("my secret text"), customKey)
if err != nil {
log.Fatal(err)
}
reader, err := service.DownloadWithURL(ctx, "gs://mybucket/folder/secret1.txt", customKey)
data, err := io.ReadAll(reader)
if err != nil {
log.Fatal(err)
}
fmt.Printf("data: %s\n", data)
- option.Checksum{Skip:true}: checksum (crc/md5) is not computed to stream data in chunks
- option.Stream: download reader reads data with specified stream PartSize
jwtConfig, err := gs.NewJwtConfig()
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
fs := afs.New()
sourceURL := "gs://myBucket/path/myasset.gz"
reader, err := fs.DownloadWithURL(ctx, sourceURL, jwtConfig, option.NewStream(64*1024*1024, 0))
if err != nil {
log.Fatal(err)
}
_ = os.Setenv("AWS_SDK_LOAD_CONFIG", "true")
destURL := "s3://myBucket/path/myasset.gz"
err = fs.Upload(ctx, destURL, 0644, reader, &option.Checksum{Skip:true})
if err != nil {
log.Fatal(err)
return
}
Documentation
¶
Overview ¶
Package gs implements abstract file storage API with Google Storage
Index ¶
- Constants
- Variables
- func GetListCounter(reset bool) int
- func GetRetryCodes(reset bool) map[int]int
- func HasAuthOption(options []option.ClientOption) bool
- func New(options ...storage.Option) storage.Manager
- func NewReadSeeker(ctx context.Context, storager *storager, call *storage.ObjectsGetCall, ...) io.ReadSeeker
- func NewStorager(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error)
- func Options(base, options []option.ClientOption) []option.ClientOption
- func Provider(options ...storage.Option) (storage.Manager, error)
- func SetCustomKeyHeader(key *option.AES256Key, headers http.Header) error
- func SetOptions(options ...goptions.ClientOption)
- type ClientOptions
- type Project
- type Scopes
Examples ¶
Constants ¶
View Source
const ( // UserAgent defines this library user agent UserAgent = "viant/gcpgs" // DefaultUploadChunkSize default upload chunk size DefaultUploadChunkSize = 16 * 1024 * 1024 )
View Source
const Scheme = "gs"
Scheme represents google storage scheme
Variables ¶
View Source
var DefaultHTTPClientProvider = getDefaultHTTPClient
DefaultHTTPClientProvider defaultHTTP client
View Source
var DefaultOptions []goptions.ClientOption
DefaultOptions represents default client option
View Source
var DefaultProjectProvider = getDefaultProject
DefaultProjectProvider default projectid provider
Functions ¶
func GetListCounter ¶ added in v0.9.4
GetListCounter returns count of list operations
func GetRetryCodes ¶ added in v0.11.2
func HasAuthOption ¶ added in v1.9.2
func HasAuthOption(options []option.ClientOption) bool
func NewReadSeeker ¶ added in v0.6.0
func NewReadSeeker(ctx context.Context, storager *storager, call *storage.ObjectsGetCall, size int) io.ReadSeeker
NewReadSeeker create a reader seeker
func NewStorager ¶
func NewStorager(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error)
NewStorager returns new storager
func Options ¶ added in v1.9.2
func Options(base, options []option.ClientOption) []option.ClientOption
func SetCustomKeyHeader ¶
SetCustomKeyHeader set encryption headers
func SetOptions ¶ added in v1.9.0
func SetOptions(options ...goptions.ClientOption)
SetOptions set global default options
Types ¶
type ClientOptions ¶
type ClientOptions []option.ClientOption
ClientOptions represents gcp client options
func NewClientOptions ¶
func NewClientOptions(options ...option.ClientOption) ClientOptions
NewClientOptions creates an option slice
Source Files
Click to show internal directories.
Click to hide internal directories.