rockset

README

Rockset Go Client

Official Go client library for Rockset

Installation

Install the Rockset Go client from Github:

go get github.com/rockset/rockset-go-client

or install it from a source code checkout:

cd $GOPATH/src/github.com
mkdir rockset
cd rockset
git clone git@github.com:rockset/rockset-go-client.git
go install rockset-go-client/rockclient.go

Usage

You can see a few examples in the godoc of how to create a collection, how to put documents in a collection and how to use SQL to query your collections.

client, err := rockset.NewClient(rockset.WithAPIKey("..."))
if err != nil {
    log.Fatal(err)
}

q := models.QueryRequest{
    Sql: &models.QueryRequestSql{
        Query: `SELECT * FROM "_events" LIMIT 1`,
    },
}

res, _, err := client.Query(q)
if err != nil {
    log.Fatal(err)
}

fmt.Printf("%+v\n", res)

Testing

Tests are available in the test folder.

Set ROCKSET_APIKEY and ROCKSET_APISERVER endpoint in the environment variables. To run tests:

go test ./...

Support

Feel free to log issues against this client through GitHub.

License

The Rockset Go Client is licensed under the Apache 2.0 License

Documentation

Overview

Package rockset provides a go client to interact with the the online Rockset database.

Uses the REST API https://docs.rockset.com/rest-api/ using an OpenAPI client, generated by the https://openapi-generator.tech/.

It provides some convenience functions to make the generated client easier to use, by wrapping common API calls in methods that require passing a context.Context and having all required arguments in the method signature, and using functional options for any optional arguments. They return the payload data from the models package, to reduce the indirection.

For example

c, err := client.GetCollection(ctx, "commons", "_events")

as opposed to the "raw" way of using the generated client code

// both workspace and collection are required arguments
p := collections.NewGetCollectionParams()
p.Workspace = "commons"
p.Collection = "_events"
p.Context = ctx
r, err := client.Collections.GetCollection(p)

If you set a zerolog logger in the context, many of the methods will log to it. E.g.

ctx := context.Background()
console := zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}
log := zerolog.New(console).Level(zerolog.TraceLevel).With().Timestamp().Logger()
ctx = log.WithContext(ctx)

Example log output

2021-05-04T07:40:01-07:00 TRC ListAllowlistIps d=565.021631 responses=0 success=true

Example (QueryRaw)

Raw usage of the openapi client

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

q := rc.QueriesApi.Query(ctx)
rq := openapi.NewQueryRequestWithDefaults()

rq.Sql = openapi.NewQueryRequestSql("SELECT * FROM commons._events where label = :label")
rq.Sql.GenerateWarnings = openapi.PtrBool(true)
rq.Sql.ProfilingEnabled = openapi.PtrBool(true)
rq.Sql.DefaultRowLimit = openapi.PtrInt32(10)

rq.Sql.Parameters = &[]openapi.QueryParameter{
	{
		Name:  "label",
		Type:  "string",
		Value: "QUERY_SUCCESS",
	},
}

r, _, err := q.Body(*rq).Execute()
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

Example (S3)

Example code to first create an S3 integration, then create a collection from the integration, and finally clean up.

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

// create integration
r, err := rc.CreateS3Integration(ctx, "s3exampleIntegration",
	option.AWSRole("arn:aws:iam::216690786812:role/rockset-integration-role"),
	option.WithS3IntegrationDescription("created by go example code"))
if err != nil {
	log.Fatalf("failed to create integration: %v", err)
}
fmt.Printf("integration created: %s\n", r.Data.GetName())

// create collection
c, err := rc.CreateS3Collection(ctx, "commons", "s3example", "created by go example code",
	"s3exampleIntegration", "rockset-terraform-provider", "cities.csv",
	rockset.WithCSVFormat(
		[]string{"city", "country", "population", "visited"},
		[]rockset.ColumnType{
			rockset.ColumnTypeSTRING, rockset.ColumnTypeSTRING, rockset.ColumnTypeInteger, rockset.ColumnTypeBool,
		},
		option.WithEncoding("UTF-8"),
		option.WithEscapeChar("\\"),
		option.WithQuoteChar(`"`),
		option.WithSeparator(","),
	),
	option.WithCollectionFieldSchema("city", option.WithColumnIndexMode(option.ColumnIndexModeNoStore)),
	option.WithCollectionFieldMapping("test", false,
		option.OutputField("out", "CAST(:country AS string)", option.OnErrorSkip),
		option.InputField("country", option.FieldMissingSkip, false, "country")),
)
if err != nil {
	var e openapi.GenericOpenAPIError
	if errors.As(err, &e) {
		log.Printf("err: %s", string(e.Body()))
	}
	log.Fatalf("failed to create collection: %v", err)
}
fmt.Printf("collection created: %s\n", c.GetName())

// wait until collection is ready
err = rc.WaitForCollectionReady(ctx, "commons", "s3example")
if err != nil {
	log.Fatalf("failed to wait for collection to be ready: %v", err)
}
fmt.Printf("collection ready: %s\n", c.GetName())

// wait until there are at least 3 new documents in the collection
err = rc.WaitForCollectionDocuments(ctx, "commons", "s3example", 3)
if err != nil {
	log.Fatalf("failed to wait for new documents: %v", err)
}

// get number of documents
collection, err := rc.GetCollection(ctx, "commons", "s3example")
if err != nil {
	log.Fatalf("failed to get collection: %v", err)
}
fmt.Printf("collection documents: %d\n", collection.Stats.GetDocCount())

// delete the collection
err = rc.DeleteCollection(ctx, "commons", "s3example")
if err != nil {
	log.Fatalf("failed to delete collection: %v", err)
}
fmt.Printf("collection deleted: %s\n", c.GetName())

// wait until the collection is gone
err = rc.WaitForCollectionGone(ctx, "commons", "s3example")
if err != nil {
	log.Fatalf("failed to wait for collection to be gone: %v", err)
}
fmt.Printf("collection gone: %s\n", c.GetName())

// delete integration
err = rc.DeleteIntegration(ctx, "s3exampleIntegration")
if err != nil {
	log.Fatalf("failed to delete integration: %v", err)
}
fmt.Printf("integration deleted: %s\n", r.Data.GetName())

Output:

integration created: s3exampleIntegration
collection created: s3example
collection ready: s3example
collection documents: 3
collection deleted: s3example
collection gone: s3example
integration deleted: s3exampleIntegration

Index

• Constants
• func IsNotFoundError(err error) bool
• type ColumnType
  • func (c ColumnType) String() string
• type Format
  • func WithCSVFormat(columnNames []string, columnTypes []ColumnType, options ...option.CSV) Format
  • func WithJSONFormat() Format
  • func WithXMLFormat(xml openapi.XmlParams) Format
• type RockClient
  • func NewClient(options ...RockOption) (*RockClient, error)
  • func (rc *RockClient) CreateDynamoDBCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateDynamoDBIntegration(ctx context.Context, name string, creds option.AWSCredentialsFn, ...) (openapi.CreateIntegrationResponse, error)
  • func (rc *RockClient) CreateFileUploadCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateGCSCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateGCSIntegration(ctx context.Context, name string, creds option.GCSCredentialsFn, ...) (openapi.CreateIntegrationResponse, error)
  • func (rc *RockClient) CreateKafkaCollection(ctx context.Context, workspace, name, description, integration, topic string, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateKinesisCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateKinesisIntegration(ctx context.Context, name string, creds option.AWSCredentialsFn, ...) (openapi.CreateIntegrationResponse, error)
  • func (rc *RockClient) CreateMongoDBCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateRedshiftCollection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateS3Collection(ctx context.Context, ...) (openapi.Collection, error)
  • func (rc *RockClient) CreateS3Integration(ctx context.Context, name string, creds option.AWSCredentialsFn, ...) (openapi.CreateIntegrationResponse, error)
  • func (rc *RockClient) DeleteCollection(ctx context.Context, workspace, name string) error
  • func (rc *RockClient) DeleteIntegration(ctx context.Context, name string) error
  • func (rc *RockClient) ExecuteQueryLambda(ctx context.Context, workspace, name string, ...) (openapi.QueryResponse, error)
  • func (rc *RockClient) GetCollection(ctx context.Context, workspace, name string) (openapi.Collection, error)
  • func (rc *RockClient) Query(ctx context.Context, sql string, options ...option.QueryOption) (openapi.QueryResponse, error)
  • func (rc *RockClient) ValidateQuery(ctx context.Context, sql string, options ...option.QueryOption) (openapi.ValidateQueryResponse, error)
  • func (rc *RockClient) WaitForCollectionDocuments(ctx context.Context, workspace, name string, count int64) error
  • func (rc *RockClient) WaitForCollectionGone(ctx context.Context, workspace, name string) error
  • func (rc *RockClient) WaitForCollectionReady(ctx context.Context, workspace, name string) error
• type RockOption
  • func Debug() RockOption
  • func FromEnv() RockOption
  • func WithAPIKey(apiKey string) RockOption
  • func WithAPIServer(server string) RockOption

Examples

• Package (QueryRaw)
• Package (S3)
• RockClient (Query)
• RockClient (QueryLambda)
• RockClient (QueryLambdaByTag)
• RockClient (QueryLambdaByVersion)
• RockClient (ValidateQuery)

Constants

const APIKeyEnvironmentVariableName = "ROCKSET_APIKEY"

const APIServerEnvironmentVariableName = "ROCKSET_APISERVER"

const DefaultAPIServer = "https://api.rs2.usw2.rockset.com"

DefaultAPIServer is the default Rockset API server to use

const LatestTag = "latest"

LatestTag is the query lambda tag for the latest version.

const Version = "0.11.0"

Version is the Rockset client version

Functions

func IsNotFoundError

func IsNotFoundError(err error) bool

Types

type ColumnType

type ColumnType int

const (
	ColumnTypeUnknown ColumnType = iota
	ColumnTypeBoolean
	ColumnTypeInteger
	ColumnTypeFloat
	ColumnTypeSTRING
	ColumnTypeTime
	ColumnTypeDate
	ColumnTypeDatetime
	ColumnTypeTimestamp
	ColumnTypeBool
	ColumnTypeInt
)

func (ColumnType) String

func (c ColumnType) String() string

String returns the string representation of the ColumnType

type Format

type Format func(params *openapi.FormatParams)

func WithCSVFormat

func WithCSVFormat(columnNames []string, columnTypes []ColumnType, options ...option.CSV) Format

func WithJSONFormat

func WithJSONFormat() Format

func WithXMLFormat

func WithXMLFormat(xml openapi.XmlParams) Format

type RockClient

type RockClient struct {
	*openapi.APIClient
}

Example (Query)

Query convenience method

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

r, err := rc.Query(ctx, "SELECT * FROM commons._events where label = :label",
	option.WithWarnings(), option.WithProfiling(), option.WithRowLimit(10),
	option.WithParameter("label", "string", "QUERY_SUCCESS"))
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

Example (QueryLambda)

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

r, err := rc.ExecuteQueryLambda(ctx, "commons", "test")
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

Example (QueryLambdaByTag)

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

r, err := rc.ExecuteQueryLambda(ctx, "commons", "test", option.WithTag("latest"))
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

Example (QueryLambdaByVersion)

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

r, err := rc.ExecuteQueryLambda(ctx, "commons", "test", option.WithVersion("foobar"))
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

Example (ValidateQuery)

ctx := context.TODO()

rc, err := rockset.NewClient(rockset.FromEnv())
if err != nil {
	log.Fatal(err)
}

r, err := rc.Query(ctx, "SELECT * FROM commons._events where label = :label",
	option.WithParameter("label", "string", "QUERY_SUCCESS"))
if err != nil {
	log.Fatal(err)
}

for _, c := range *r.Collections {
	fmt.Printf("collection: %s\n", c)
}

Output:

collection: commons._events

func NewClient

func NewClient(options ...RockOption) (*RockClient, error)

NewClient creates a new Rockset client.

Accessing the online database requires an API key, which you either have to supply through the ROCKSET_APIKEY environment variable and pass the FromEnv() option

c, err := rockset.NewClient(rockset.FromEnv())

or explicitly using the WithAPIKey() option

c, err := rockset.NewClient(rockset.WithAPIKey("..."))

func (*RockClient) CreateDynamoDBCollection

func (rc *RockClient) CreateDynamoDBCollection(ctx context.Context,
	workspace, name, description, integration, region, tableName string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateDynamoDBIntegration

func (rc *RockClient) CreateDynamoDBIntegration(ctx context.Context, name string, creds option.AWSCredentialsFn,
	options ...option.DynamoDBIntegrationOption) (openapi.CreateIntegrationResponse, error)

func (*RockClient) CreateFileUploadCollection

func (rc *RockClient) CreateFileUploadCollection(ctx context.Context,
	workspace, name, description, integration, fileName string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateGCSCollection

func (rc *RockClient) CreateGCSCollection(ctx context.Context,
	workspace, name, description, integration, bucket, prefix string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateGCSIntegration

func (rc *RockClient) CreateGCSIntegration(ctx context.Context, name string, creds option.GCSCredentialsFn,
	options ...option.GCSIntegrationOption) (openapi.CreateIntegrationResponse, error)

func (*RockClient) CreateKafkaCollection

func (rc *RockClient) CreateKafkaCollection(ctx context.Context,
	workspace, name, description, integration, topic string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateKinesisCollection

func (rc *RockClient) CreateKinesisCollection(ctx context.Context,
	workspace, name, description, integration, region, stream string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateKinesisIntegration

func (rc *RockClient) CreateKinesisIntegration(ctx context.Context, name string, creds option.AWSCredentialsFn,
	options ...option.KinesisIntegrationOption) (openapi.CreateIntegrationResponse, error)

func (*RockClient) CreateMongoDBCollection

func (rc *RockClient) CreateMongoDBCollection(ctx context.Context,
	workspace, name, description, integration, database, collection string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateRedshiftCollection

func (rc *RockClient) CreateRedshiftCollection(ctx context.Context,
	workspace, name, description, integration, database, schema, tableName string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

func (*RockClient) CreateS3Collection

func (rc *RockClient) CreateS3Collection(ctx context.Context,
	workspace, name, description, integration, bucket, pattern string,
	format Format, options ...option.CollectionOption) (openapi.Collection, error)

CreateS3Collection creates an S3 collection from an existing S3 integration. Not specifying a format will default to JSON.

func (*RockClient) CreateS3Integration

func (rc *RockClient) CreateS3Integration(ctx context.Context, name string, creds option.AWSCredentialsFn,
	options ...option.S3IntegrationOption) (openapi.CreateIntegrationResponse, error)

func (*RockClient) DeleteCollection

func (rc *RockClient) DeleteCollection(ctx context.Context, workspace, name string) error

func (*RockClient) DeleteIntegration

func (rc *RockClient) DeleteIntegration(ctx context.Context, name string) error

func (*RockClient) ExecuteQueryLambda

func (rc *RockClient) ExecuteQueryLambda(ctx context.Context, workspace, name string,
	options ...option.QueryLambdaOption) (openapi.QueryResponse, error)

ExecuteQueryLambda executes a query lambda with optional query options.

func (*RockClient) GetCollection

func (rc *RockClient) GetCollection(ctx context.Context, workspace, name string) (openapi.Collection, error)

func (*RockClient) Query

func (rc *RockClient) Query(ctx context.Context, sql string,
	options ...option.QueryOption) (openapi.QueryResponse, error)

func (*RockClient) ValidateQuery

func (rc *RockClient) ValidateQuery(ctx context.Context, sql string,
	options ...option.QueryOption) (openapi.ValidateQueryResponse, error)

func (*RockClient) WaitForCollectionDocuments

func (rc *RockClient) WaitForCollectionDocuments(ctx context.Context, workspace, name string, count int64) error

WaitForCollectionDocuments waits until the collection has at least count new documents

func (*RockClient) WaitForCollectionGone

func (rc *RockClient) WaitForCollectionGone(ctx context.Context, workspace, name string) error

WaitForCollectionGone waits until the a collection marked for deletion is gone, i.e. GetCollection() returns "not found".

func (*RockClient) WaitForCollectionReady

func (rc *RockClient) WaitForCollectionReady(ctx context.Context, workspace, name string) error

WaitForCollectionReady waits until the collection is ready.

type RockOption

type RockOption func(rc *openapi.Configuration)

RockOption is the type for RockClient options

func Debug

func Debug() RockOption

func FromEnv

func FromEnv() RockOption

FromEnv sets API key and API server from the environment variables ROCKSET_APIKEY and ROCKSET_APISERVER, and if ROCKSET_APISERVER is not set, it will use the default API server.

func WithAPIKey

func WithAPIKey(apiKey string) RockOption

func WithAPIServer

func WithAPIServer(server string) RockOption