td_client

package module
v0.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 10, 2023 License: Apache-2.0 Imports: 19 Imported by: 9

README

Treasure Data API client for Go

Build Status

Build

  • make
    • To build code and run all the tests
  • make test-verbose
    • To run tests verbosely
  • make help
    • To describe targets
  • make clean
    • To cleanup locally

Synopsis

package main

import (
	td_client "github.com/treasure-data/td-client-go"
)

func main() {
	client, err := td_client.NewTDClient(td_client.Settings {
		ApiKey: "YOUR-API-KEY-HERE",
	})
	if err != nil { ... }
	jobId, err := client.SubmitQuery("mydatabase", td_client.Query {
		Type: "hive",
		Query: "SELECT * FROM mytable WHERE value >= 500",
	})
	if err != nil { ... }
	for {
		status, err := client.JobStatus(jobId)
		if err != nil { ... }
		if status != "queued" && status != "running" { break }
		time.Sleep(1000000000)
	}
	err = client.JobResultEach(jobId, func(v interface{}) error {
		fmt.Printf("Result:%v\n", v)
		return nil
	})
	if err != nil { ... }
}

License

Copyright (C) 2014 Treasure Data, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Documentation

Overview

Treasure Data API client. The following API functions are covered at the moment.

Account API: 

ShowAccount
AccountCoreUtilization

Status API: 

ServerStatus

Database/Table API: 

ListDatabases
DeleteDatabase
CreateDatabase
ListTables
CreateTable
DeleteTable
SwapTable
UpdateSchema
UpdateExpire
Tail

Job/Query API: 

ListJobs
ShowJob
KillJob
SubmitQuery
SubmitExportJob
SubmitPartialDeleteJob
JobStatus
JobResult
ListResults
CreateResult
DeleteResult

Import API: 

Import

Index

Constants

View Source 
const (
	GenericError = iota
	AuthError
	ForbiddenError
	AlreadyExistsError
	NotFoundError
)
View Source 
const (
	DEFAULT_ENDPOINT            = "api.treasure-data.com"
	DEFAULT_IMPORT_ENDPOINT     = "api-import.treasure-data.com"
	NEW_DEFAULT_ENDPOINT        = "api.treasuredata.com"
	NEW_DEFAULT_IMPORT_ENDPOINT = "api-import.treasuredata.com"
)
View Source 
const (
	// Represents the date/time format for time.Time.Format(),
	// which is used in several API function parameters and results.
	TDAPIDateTime            = "2006-01-02 15:04:05 MST"
	TDAPIDateTimeNumericZone = "2006-01-02 15:04:05 -0700"
)
View Source 
const (
	CLIENT_VERSION = "0.3.0"
)

Variables

View Source 
var DefaultRouter = V3EndpointRouter{
	DefaultEndpoint: NEW_DEFAULT_ENDPOINT,
	ImportEndpoint:  NEW_DEFAULT_IMPORT_ENDPOINT,
}

DefaultRouter is a V3EndpointRouter with the hard-coded endpoints.

Functions

This section is empty.

Types

type APIError 

type APIError struct {
	Type    int
	Message string
	Cause   error
}

APIError represents an error that has occurred during the API call.

func (*APIError) Error 

func (e *APIError) Error() string

type AddAPIKeyResult 

type AddAPIKeyResult struct {
	APIKey string
}

AddAPIKeyResult represents the result of AddAPIKey API

type AuthenticateResult 

type AuthenticateResult struct {
	Name   string
	APIKey string
}

AuthenticateResult is result of authenticate API

type Blob 

type Blob interface {
	Reader() (io.ReadCloser, error)
	Size() (int64, error)
	MD5Sum() ([]byte, error)
}

Blob denotes a concept, which is opaque data that can be read bytewise through an io.Reader, has a certain size and provides a calculated MD5 sum.

func NewBufferingBlob 

func NewBufferingBlob(blob Blob) Blob

func NewBufferingBlobSize 

func NewBufferingBlobSize(blob Blob, size int) Blob

type BufferingBlob 

type BufferingBlob struct {
	// contains filtered or unexported fields
}

BufferingBlob wraps the other blob so Reader() would return the buffered reader. This is helpful if the blob is backed by a *os.File.

func (*BufferingBlob) MD5Sum 

func (blob *BufferingBlob) MD5Sum() ([]byte, error)

func (*BufferingBlob) Reader 

func (blob *BufferingBlob) Reader() (io.ReadCloser, error)

func (*BufferingBlob) Size 

func (blob *BufferingBlob) Size() (int64, error)

type BulkImportElement 

type BulkImportElement struct {
	Name         string
	Database     string
	Table        string
	Status       string
	JobID        string
	ValidRecords int
	ErrorRecords int
	ValidParts   int
	ErrorParts   int
	UploadFrozen bool
}

type BulkImportResult 

type BulkImportResult struct {
	Name       string
	BulkImport string
}

type ConverterFunc 

type ConverterFunc func(string) (interface{}, error)

Used in internal schema, marking the field so that it will be unmarshaled by the specified function.

func EmbeddedJSON 

func EmbeddedJSON(expectedTypeProto interface{}) ConverterFunc

EmbeddedJSON is a factory used internally that makes a ConverterFunc function that returns the specified type.

type DeleteScheduleResult 

type DeleteScheduleResult struct {
	Name      string
	Cron      string
	Type      string
	Query     string
	Timezone  string
	Delay     int
	Database  string
	UserName  string
	CreatedAt time.Time
}

type EndpointRouter 

type EndpointRouter interface {
	Route(requestUri string) string
}

EndpointRouter is expected to return the host name most suitable for the passed request URI

type FixedEndpointRouter 

type FixedEndpointRouter struct {
	Endpoint string
}

A FixedEndpointRouter instance represents an EndpointRouter that always routes the request to the same endpoint.

func (*FixedEndpointRouter) Route 

func (r *FixedEndpointRouter) Route(_ string) string

type InMemoryBlob 

type InMemoryBlob []byte

InMemoryBlob is a Blob which stores the entire data as a byte array.

func (InMemoryBlob) MD5Sum 

func (b InMemoryBlob) MD5Sum() ([]byte, error)

func (InMemoryBlob) Reader 

func (b InMemoryBlob) Reader() (io.ReadCloser, error)

func (InMemoryBlob) Size 

func (b InMemoryBlob) Size() (int64, error)

type ListAPIKeysResult 

type ListAPIKeysResult struct {
	APIKeys []string
}

ListAPIKeysResult represents the result of ListAPIKeys API

type ListBulkImportElements 

type ListBulkImportElements []BulkImportElement

type ListBulkImportParts 

type ListBulkImportParts struct {
	Name       string
	BulkImport string
	Parts      []string
}

type ListDataBasesResult 

type ListDataBasesResult []ListDataBasesResultElement

ListDataBasesResult is a collection of ListDataBasesResultElement

type ListDataBasesResultElement 

type ListDataBasesResultElement struct {
	Name            string
	Organization    string
	Count           int
	CreatedAt       time.Time
	UpdatedAt       time.Time
	Permission      string
	DeleteProtected bool
}

ListDataBasesResultElement represents an item of the result of ListDatabases API call

type ListJobsOptions 

type ListJobsOptions struct {
	// contains filtered or unexported fields
}

func (*ListJobsOptions) WithFrom 

func (options *ListJobsOptions) WithFrom(from int) *ListJobsOptions

func (*ListJobsOptions) WithStatus 

func (options *ListJobsOptions) WithStatus(status string) *ListJobsOptions

func (*ListJobsOptions) WithTo 

func (options *ListJobsOptions) WithTo(to int) *ListJobsOptions

type ListJobsResult 

type ListJobsResult struct {
	ListJobsResultElements ListJobsResultElements
	Count                  int
	From                   int
	To                     int
}

type ListJobsResultElement 

type ListJobsResultElement struct {
	Id         string
	Type       string
	Database   string
	Status     string
	Query      string
	Duration   int
	CreatedAt  time.Time
	UpdatedAt  time.Time
	StartAt    time.Time
	EndAt      time.Time
	CpuTime    float64
	ResultSize int
	NumRecords int
	ResultUrl  string
	Priority   int
	RetryLimit int
}

type ListJobsResultElements 

type ListJobsResultElements []ListJobsResultElement

type ListResultsResult 

type ListResultsResult []ListResultsResultElement

type ListResultsResultElement 

type ListResultsResultElement struct {
	Name string
	Url  string
}

type ListScheduleResult 

type ListScheduleResult []ScheduleElement

type ListTablesResult 

type ListTablesResult []ListTablesResultElement

ListTablesResult is a collection of ListTablesResultElement

type ListTablesResultElement 

type ListTablesResultElement struct {
	Id                   int
	Name                 string
	Type                 string
	Count                int
	CreatedAt            time.Time
	UpdatedAt            time.Time
	LastImport           time.Time
	LastLogTimestamp     time.Time
	EstimatedStorageSize int
	Schema               []interface{}
	ExpireDays           int
	PrimaryKey           string
	PrimaryKeyType       string
	IncludeV             bool
}

ListTablesResultElement represents an item of the result of ListTables API call

type ListUsersResult 

type ListUsersResult []ListUsersResultElement

ListUsersResult is a collection of ListUsersResultElement

type ListUsersResultElement 

type ListUsersResultElement struct {
	ID            int
	FirstName     string
	LastName      string
	Email         string
	Phone         string
	GravatarURL   string
	Administrator bool
	CreatedAt     time.Time
	UpdatedAt     time.Time
	Name          string
	AccountOwner  bool
	Organization  string
	Roles         []string
}

ListUsersResultElement represents an item of the result of ListUsers API

type Optional 

type Optional struct {
	V       interface{}
	Default interface{}
}

Used in internal schema, marking the field as optional as well as providing the default.

type PerformBulkImportResult 

type PerformBulkImportResult struct {
	Name       string
	BulkImport string
	JobID      string
}

type Query 

type Query struct {
	Type          string
	Query         string
	ResultUrl     string
	Priority      int
	RetryLimit    int
	EngineVersion string
}

type RunScheduleResult 

type RunScheduleResult struct {
	ID          string
	Type        string
	ScheduledAt time.Time
}

type RunScheduleResultList 

type RunScheduleResultList []RunScheduleResult

type ScheduleElement 

type ScheduleElement struct {
	Name       string
	Cron       string
	Type       string
	Query      string
	Timezone   string
	Delay      int
	Database   string
	UserName   string
	Priority   int
	RetryLimit int
	Result     string
	NextTime   string
	CreatedAt  time.Time
}

type ScheduleHistoryElement 

type ScheduleHistoryElement struct {
	ID               string
	Query            string
	Type             string
	URL              string
	Database         string
	Status           string
	StartAt          time.Time
	EndAt            time.Time
	ScheduledAt      time.Time
	CreatedAt        time.Time
	UpdatedAt        time.Time
	UserName         string
	CPUTime          float64
	Duration         float64
	ResultSize       int
	NumRecords       int
	Result           string
	Priority         int
	RetryLimit       int
	HiveResultSchema []interface{}
	Organization     string
}

type ScheduleHistoryElementList 

type ScheduleHistoryElementList []ScheduleHistoryElement

type ScheduleHistoryList 

type ScheduleHistoryList struct {
	History ScheduleHistoryElementList
	Count   int
	From    int
	To      int
}

type ScheduleResult 

type ScheduleResult struct {
	ID         string
	Name       string
	Cron       string
	Type       string
	Query      string
	Timezone   string
	Delay      int
	Database   string
	UserName   string
	Priority   int
	RetryLimit int
	Result     string
	Start      string
	CreatedAt  time.Time
}

type ServerStatusResult 

type ServerStatusResult struct {
	Status string
}

type Settings 

type Settings struct {
	ApiKey            string            // Treasure Data Account API key
	UserAgent         string            // (Optional) Name that will appear as the User-Agent HTTP header
	Router            EndpointRouter    // (Optional) Endpoint router
	ConnectionTimeout time.Duration     // (Optional) Connection timeout
	ReadTimeout       time.Duration     // (Optional) Read timeout.
	SendTimeout       time.Duration     // (Optional) Send timeout.
	RootCAs           *x509.CertPool    // (Optional) Specify the CA certificates.
	Port              int               // (Optional) Port number.
	Proxy             interface{}       // (Optional) HTTP proxy to use.
	Transport         http.RoundTripper // (Optional) Overrides the transport used to establish the connection.
	Headers           map[string]string // (Optional) Additional headers that will be sent to the endpoint.
}

Settings stores the parameters for initializaing TDClient.

Note that ReadTimeout / SendTimeout includes the time taken for receiving / sending the actual data in addition to the idle time, so it is advised to set the value long enough depending on the circumstances. (network latency etc.)

Specifying 0 to Port means the value will be automatically determined according to the settings.

Proxy can take three kinds of values: *url.URL (parsed URL), func(*http.Request)(*url.URL, error), string (URL) or nil (the direct connection to the endpoint is possible).

Transport allows you to take more control over the communication.

`Ssl` option was removed from client options. td-client-go no longer support `Ssl` option since Treasure Data permits only HTTPS access after September 1, 2020.

type ShowAccountResult 

type ShowAccountResult struct {
	Id              int
	Plan            int
	StorageSize     int
	GuaranteedCores int
	MaximumCores    int
	CreatedAt       time.Time
}

ShowAccountResult stores the result of `ShowAccountResult` API call

type ShowJobResult 

type ShowJobResult struct {
	Id               string
	Type             string
	Database         string
	UserName         string
	Status           string
	Query            string
	Debug            ShowJobResultDebugElement
	Url              string
	Duration         int
	CreatedAt        time.Time
	UpdatedAt        time.Time
	StartAt          time.Time
	EndAt            time.Time
	CpuTime          float64
	ResultSize       int
	NumRecords       int
	ResultUrl        string
	Priority         int
	RetryLimit       int
	HiveResultSchema []interface{}
}

ShowJobResult stores the result of `ShowJobResult` API call.

type ShowJobResultDebugElement 

type ShowJobResultDebugElement struct {
	CmdOut string
	StdErr string
}

type TDClient 

type TDClient struct {
	// contains filtered or unexported fields
}

TDClient represents a context used to talk to the Treasure Data API.

func NewTDClient 

func NewTDClient(settings Settings) (*TDClient, error)

Creates a new TDClient instance according to the settings.

func (*TDClient) AddAPIKey 

func (client *TDClient) AddAPIKey(email string) (*AddAPIKeyResult, error)

func (*TDClient) AddUser 

func (client *TDClient) AddUser(name, org, email, password string) error

func (*TDClient) Authenticate 

func (client *TDClient) Authenticate(email, password string) (*AuthenticateResult, error)

func (*TDClient) CommitBulkImport 

func (client *TDClient) CommitBulkImport(name string, options map[string]string) (*BulkImportResult, error)

func (*TDClient) CreateBulkImport 

func (client *TDClient) CreateBulkImport(name string, db string, table string, options map[string]string) (*BulkImportResult, error)

func (*TDClient) CreateDatabase 

func (client *TDClient) CreateDatabase(db string, options map[string]string) error

func (*TDClient) CreateLogTable 

func (client *TDClient) CreateLogTable(db string, table string) error

func (*TDClient) CreateResult 

func (client *TDClient) CreateResult(name, url_ string) error

func (*TDClient) CreateSchedule 

func (client *TDClient) CreateSchedule(scheduleName string, options map[string]string) (*ScheduleResult, error)

func (*TDClient) DeleteBulkImport 

func (client *TDClient) DeleteBulkImport(name string, options map[string]string) error

func (*TDClient) DeleteBulkImportPart 

func (client *TDClient) DeleteBulkImportPart(name string, part_name string, options map[string]string) error

func (*TDClient) DeleteDatabase 

func (client *TDClient) DeleteDatabase(db string) error

func (*TDClient) DeleteResult 

func (client *TDClient) DeleteResult(name string) error

func (*TDClient) DeleteSchedule 

func (client *TDClient) DeleteSchedule(scheduleName string) (*DeleteScheduleResult, error)

func (*TDClient) DeleteTable 

func (client *TDClient) DeleteTable(db string, table string) (string, error)

func (*TDClient) FreezeBulkImport 

func (client *TDClient) FreezeBulkImport(name string, options map[string]string) (*BulkImportResult, error)

func (*TDClient) Import 

func (client *TDClient) Import(db string, table string, format string, blob Blob, uniqueId string) (float64, error)

`Import` API call.

func (*TDClient) JobResult 

func (client *TDClient) JobResult(jobId string, format string, reader func(io.Reader) error) error

func (*TDClient) JobResultEach 

func (client *TDClient) JobResultEach(jobId string, reader func(interface{}) error) error

func (*TDClient) JobStatus 

func (client *TDClient) JobStatus(jobId string) (string, error)

func (*TDClient) KillJob 

func (client *TDClient) KillJob(jobId string) error

func (*TDClient) ListAPIKeys 

func (client *TDClient) ListAPIKeys(email string) (*ListAPIKeysResult, error)

func (*TDClient) ListBulkImportParts 

func (client *TDClient) ListBulkImportParts(name string, options map[string]string) (*ListBulkImportParts, error)

func (*TDClient) ListBulkImports 

func (client *TDClient) ListBulkImports(options map[string]string) (*ListBulkImportElements, error)

func (*TDClient) ListDatabases 

func (client *TDClient) ListDatabases() (*ListDataBasesResult, error)

func (*TDClient) ListJobs 

func (client *TDClient) ListJobs() (*ListJobsResult, error)

func (*TDClient) ListJobsWithOptions 

func (client *TDClient) ListJobsWithOptions(options *ListJobsOptions) (*ListJobsResult, error)

func (*TDClient) ListResults 

func (client *TDClient) ListResults() (*ListResultsResult, error)

func (*TDClient) ListSchedules 

func (client *TDClient) ListSchedules() (*ListScheduleResult, error)

func (*TDClient) ListTables 

func (client *TDClient) ListTables(db string) (*ListTablesResult, error)

func (*TDClient) ListUsers 

func (client *TDClient) ListUsers() (*ListUsersResult, error)

func (*TDClient) PerformBulkImport 

func (client *TDClient) PerformBulkImport(name string, options map[string]string) (*PerformBulkImportResult, error)

func (*TDClient) RemoveAPIKey 

func (client *TDClient) RemoveAPIKey(email, apikey string) error

func (*TDClient) RemoveUser 

func (client *TDClient) RemoveUser(email string) error

func (*TDClient) RunSchedule 

func (client *TDClient) RunSchedule(scheduleName string, runTime string, options map[string]string) (*RunScheduleResultList, error)

func (*TDClient) ScheduleHistory 

func (client *TDClient) ScheduleHistory(scheduleName string, options map[string]string) (*ScheduleHistoryList, error)

func (*TDClient) ServerStatus 

func (client *TDClient) ServerStatus() (*ServerStatusResult, error)

func (*TDClient) ShowAccount 

func (client *TDClient) ShowAccount() (*ShowAccountResult, error)

ShowAccount returns the information about the current account

func (*TDClient) ShowBulkImport 

func (client *TDClient) ShowBulkImport(name string) (*BulkImportElement, error)

func (*TDClient) ShowDatabase 

func (client *TDClient) ShowDatabase(dbname string) (*ListDataBasesResultElement, error)

func (*TDClient) ShowJob 

func (client *TDClient) ShowJob(jobId string) (*ShowJobResult, error)

func (*TDClient) ShowTable 

func (client *TDClient) ShowTable(db, table string) (*ListTablesResultElement, error)

func (*TDClient) SubmitExportJob 

func (client *TDClient) SubmitExportJob(db string, table string, storageType string, options map[string]string) (string, error)

func (*TDClient) SubmitPartialDeleteJob 

func (client *TDClient) SubmitPartialDeleteJob(db string, table string, to time.Time, from time.Time, options map[string]string) (string, error)

func (*TDClient) SubmitQuery 

func (client *TDClient) SubmitQuery(db string, q Query) (string, error)

func (*TDClient) SwapTable 

func (client *TDClient) SwapTable(db string, table1 string, table2 string) error

func (*TDClient) Tail 

func (client *TDClient) Tail(db string, table string, count int, to time.Time, from time.Time, reader func(interface{}) error) error

func (*TDClient) UnfreezeBulkImport 

func (client *TDClient) UnfreezeBulkImport(name string, options map[string]string) (*BulkImportResult, error)

func (*TDClient) UpdateExpire 

func (client *TDClient) UpdateExpire(db string, table string, expireDays int) error

func (*TDClient) UpdateSchedule 

func (client *TDClient) UpdateSchedule(scheduleName string, options map[string]string) (*ScheduleResult, error)

func (*TDClient) UpdateSchema 

func (client *TDClient) UpdateSchema(db string, table string, schema []interface{}) error

func (*TDClient) UpdateTable 

func (client *TDClient) UpdateTable(db string, table string, params map[string]string) error

func (*TDClient) UploadBulkImportPart 

func (client *TDClient) UploadBulkImportPart(name string, part_name string, blob Blob) (*BulkImportResult, error)

type TimeoutConn 

type TimeoutConn struct {
	Conn         net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
	// contains filtered or unexported fields
}

TimeoutConn wraps a regular net.Conn so read / write operations on it will time out in the specified amount of time.

func (*TimeoutConn) Close 

func (conn *TimeoutConn) Close() error

func (*TimeoutConn) LocalAddr 

func (conn *TimeoutConn) LocalAddr() net.Addr

func (*TimeoutConn) Read 

func (conn *TimeoutConn) Read(b []byte) (int, error)

func (*TimeoutConn) RemoteAddr 

func (conn *TimeoutConn) RemoteAddr() net.Addr

func (*TimeoutConn) SetDeadline 

func (conn *TimeoutConn) SetDeadline(t time.Time) error

func (*TimeoutConn) SetReadDeadline 

func (conn *TimeoutConn) SetReadDeadline(t time.Time) error

func (*TimeoutConn) SetWriteDeadline 

func (conn *TimeoutConn) SetWriteDeadline(t time.Time) error

func (*TimeoutConn) Write 

func (conn *TimeoutConn) Write(b []byte) (int, error)

type V3EndpointRouter 

type V3EndpointRouter struct {
	DefaultEndpoint string
	ImportEndpoint  string
}

V3EndpointRouter routes the import request to the dedicated endpoint and other requests to the default.

func (*V3EndpointRouter) Route 

func (r *V3EndpointRouter) Route(requestUri string) string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.