statscopy

package
v1.22.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 13, 2023 License: MIT Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrRequiredSpannerClient = errors.New("required spanner client")
)
View Source
var LockStatsBigQueryTableSchema = bigquery.Schema{
	{Name: "interval_end", Required: true, Type: bigquery.TimestampFieldType},
	{Name: "row_range_start_key", Required: true, Type: bigquery.BytesFieldType},
	{Name: "lock_wait_seconds", Required: true, Type: bigquery.NumericFieldType},
	{Name: "sample_lock_requests",
		Required: true,
		Repeated: true,
		Type:     bigquery.RecordFieldType,
		Schema: bigquery.Schema{
			{Name: "lock_mode", Required: true, Type: bigquery.StringFieldType},
			{Name: "column", Required: true, Type: bigquery.StringFieldType},
		}},
}

LockStatsBigQueryTableSchema is BigQuery Table Schema

View Source
var QueryStatsBigQueryTableSchema = bigquery.Schema{
	{Name: "interval_end", Required: true, Type: bigquery.TimestampFieldType},
	{Name: "text", Required: true, Type: bigquery.StringFieldType},
	{Name: "text_truncated", Required: true, Type: bigquery.BooleanFieldType},
	{Name: "text_fingerprint", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "execution_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "avg_latency_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_rows", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_bytes", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_rows_scanned", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_cpu_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "all_failed_execution_count", Required: false, Type: bigquery.IntegerFieldType},
	{Name: "all_failed_avg_latency_seconds", Required: false, Type: bigquery.FloatFieldType},
	{Name: "cancelled_or_disconnected_execution_count", Required: false, Type: bigquery.IntegerFieldType},
	{Name: "timed_out_execution_count", Required: false, Type: bigquery.IntegerFieldType},
}

QueryStatsBigQueryTableSchema is BigQuery Table Schema

View Source
var QueryStatsBigQueryTableSchema20210113 = bigquery.Schema{
	{Name: "interval_end", Required: true, Type: bigquery.TimestampFieldType},
	{Name: "text", Required: true, Type: bigquery.StringFieldType},
	{Name: "text_truncated", Required: true, Type: bigquery.BooleanFieldType},
	{Name: "text_fingerprint", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "execution_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "avg_latency_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_rows", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_bytes", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_rows_scanned", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_cpu_seconds", Required: true, Type: bigquery.FloatFieldType},
}

QueryStatsBigQueryTableSchemaV1 is BigQuery Table Schema 2021/01/14 UPDATEでFailed系のColumnが追加される前のSchema

View Source
var ReadStatsBigQueryTableSchema = bigquery.Schema{
	{Name: "interval_end", Required: true, Type: bigquery.TimestampFieldType},
	{Name: "read_columns", Required: true, Repeated: true, Type: bigquery.StringFieldType},
	{Name: "fprint", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "execution_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "avg_rows", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_bytes", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_cpu_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_locking_delay_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_client_wait_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_leader_refresh_delay_seconds", Required: true, Type: bigquery.FloatFieldType},
}

ReadStatsBigQueryTableSchema is BigQuery Table Schema

View Source
var TxStatsBigQueryTableSchema = bigquery.Schema{
	{Name: "interval_end", Required: true, Type: bigquery.TimestampFieldType},
	{Name: "fprint", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "read_columns", Required: true, Repeated: true, Type: bigquery.StringFieldType},
	{Name: "write_constructive_columns", Required: true, Repeated: true, Type: bigquery.StringFieldType},
	{Name: "write_delete_tables", Required: true, Repeated: true, Type: bigquery.StringFieldType},
	{Name: "commit_attempt_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "commit_abort_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "commit_retry_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "commit_failed_precondition_count", Required: true, Type: bigquery.IntegerFieldType},
	{Name: "avg_participants", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_total_latency_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_commit_latency_seconds", Required: true, Type: bigquery.FloatFieldType},
	{Name: "avg_bytes", Required: true, Type: bigquery.FloatFieldType},
}

TxStatsBigQueryTableSchema is BigQuery Table Schema

Functions

This section is empty.

Types

type LockStat added in v1.11.0

type LockStat struct {
	// End of the time interval that the included query executions occurred in.
	IntervalEnd time.Time `spanner:"interval_end"`

	/*
	  The row key where the lock conflict occurred.
	  When the conflict involves a range of rows, this value represents the starting key of that range.
	  A plus sign, +, signifies a range. For more information, see What's a row range start key.
	*/
	RowRangeStartKey []byte `spanner:"row_range_start_key"`

	/*
	  The cumulative lock wait time of lock conflicts recorded for all the columns in the row key range, in seconds.
	*/
	LockWaitSeconds float64 `spanner:"lock_wait_seconds"`

	/*
	  Each entry in this array corresponds to a sample lock request that contributed to the lock conflict on the given row key, or row key range.
	  The maximum number of samples in this array is 20.
	  Each sample contains the following two fields:
	    lock_mode: The lock mode that was requested.
	               For more information, see Lock modes
	    column: The column which encountered the lock conflict.
	            The format of this value is tablename.columnname.
	*/
	SampleLockRequests []*LockStatSampleLockRequest `spanner:"sample_lock_requests"`
}

func (*LockStat) InsertID added in v1.11.0

func (s *LockStat) InsertID() (string, error)

InsertID is 同じデータをBigQueryになるべく入れないようにデータからInsertIDを作成する

func (*LockStat) Save added in v1.11.0

func (s *LockStat) Save() (map[string]bigquery.Value, string, error)

Save is bigquery.ValueSaver interface

type LockStatSampleLockRequest added in v1.11.0

type LockStatSampleLockRequest struct {
	LockMode string `spanner:"lock_mode"`
	Column   string `spanner:"column"`
}

func (*LockStatSampleLockRequest) ToBQValue added in v1.13.1

func (s *LockStatSampleLockRequest) ToBQValue() map[string]bigquery.Value

Save is bigquery.ValueSaver interface

type LockStatsParam added in v1.11.0

type LockStatsParam struct {
	Table string
}

type LockStatsTopTable added in v1.11.0

type LockStatsTopTable string
const (
	LockStatsTopMinuteTable   LockStatsTopTable = "spanner_sys.lock_stats_top_minute"
	LockStatsTop10MinuteTable LockStatsTopTable = "spanner_sys.lock_stats_top_10minute"
	LockStatsTopHourTable     LockStatsTopTable = "spanner_sys.lock_stats_top_hour"
)

type QueryStat

type QueryStat struct {
	IntervalEnd                           time.Time `spanner:"interval_end"` // End of the time interval that the included query executions occurred in.
	Text                                  string    // SQL query text, truncated to approximately 64KB.
	TextTruncated                         bool      `spanner:"text_truncated"`                            // Whether or not the query text was truncated.
	TextFingerprint                       int64     `spanner:"text_fingerprint"`                          // Hash of the query text.
	ExecuteCount                          int64     `spanner:"execution_count"`                           // Number of times Cloud Spanner saw the query during the interval.
	AvgLatencySeconds                     float64   `spanner:"avg_latency_seconds"`                       // Average length of time, in seconds, for each query execution within the database. This average excludes the encoding and transmission time for the result set as well as overhead.
	AvgRows                               float64   `spanner:"avg_rows"`                                  // Average number of rows that the query returned.
	AvgBytes                              float64   `spanner:"avg_bytes"`                                 // Average number of data bytes that the query returned, excluding transmission encoding overhead.
	AvgRowsScanned                        float64   `spanner:"avg_rows_scanned"`                          // Average number of rows that the query scanned, excluding deleted values.
	AvgCPUSeconds                         float64   `spanner:"avg_cpu_seconds"`                           // Average number of seconds of CPU time Cloud Spanner spent on all operations to execute the query.
	AllFailedExecutionCount               int64     `spanner:"all_failed_execution_count"`                // Number of times the query failed during the interval.
	AllFailedAvgLatencySeconds            float64   `spanner:"all_failed_avg_latency_seconds"`            // Average length of time, in seconds, for each query execution that failed within the database. This average excludes the encoding and transmission time for the result set as well as overhead.
	CancelledOrDisconnectedExecutionCount int64     `spanner:"cancelled_or_disconnected_execution_count"` // Number of times the query was canceled by the user or failed due to broken network connection during the interval.
	TimedOutExecutionCount                int64     `spanner:"timed_out_execution_count"`                 // Number of times the query timed out during the interval.
}

func (*QueryStat) InsertID

func (s *QueryStat) InsertID() (string, error)

InsertID is 同じデータをBigQueryになるべく入れないようにデータからInsertIDを作成する

func (*QueryStat) Save

func (s *QueryStat) Save() (map[string]bigquery.Value, string, error)

Save is bigquery.ValueSaver interface

type QueryStatsParam

type QueryStatsParam struct {
	Table string
}

type QueryStatsTopTable

type QueryStatsTopTable string
const (
	QueryStatsTopMinuteTable   QueryStatsTopTable = "spanner_sys.query_stats_top_minute"
	QueryStatsTop10MinuteTable QueryStatsTopTable = "spanner_sys.query_stats_top_10minute"
	QueryStatsTopHourTable     QueryStatsTopTable = "spanner_sys.query_stats_top_hour"
)

type ReadStat

type ReadStat struct {
	IntervalEnd                  time.Time `spanner:"interval_end"` // End of the time interval that the included query executions occurred in.
	ReadColumns                  []string  `spanner:"read_columns"` // The set of columns that were read. These are in alphabetical order.
	Fprint                       int64     // Hash of the read column names.
	ExecutionCount               int64     `spanner:"execution_count"`                  // Number of times Cloud Spanner executed the read shape during the interval.
	AvgRows                      float64   `spanner:"avg_rows"`                         // Average number of rows that the read returned.
	AvgBytes                     float64   `spanner:"avg_bytes"`                        // Average number of data bytes that the read returned, excluding transmission encoding overhead.
	AvgCPUSeconds                float64   `spanner:"avg_cpu_seconds"`                  // Average number of Cloud Spanner server side CPU seconds executing the read, excluding prefetch CPU and other overhead.
	AvgLockingDelaySeconds       float64   `spanner:"avg_locking_delay_seconds"`        // Average number of seconds spent waiting due to locking.
	AvgClientWaitSeconds         float64   `spanner:"avg_client_wait_seconds"`          // Average number of seconds spent waiting due to the client not consuming data as fast as Cloud Spanner could generate it.
	AvgLeaderRefreshDelaySeconds float64   `spanner:"avg_leader_refresh_delay_seconds"` // Average number of seconds spent waiting to confirm with the Paxos leader that all writes have been observed..
}

func (*ReadStat) InsertID

func (s *ReadStat) InsertID() (string, error)

InsertID is 同じデータをBigQueryになるべく入れないようにデータからInsertIDを作成する

func (*ReadStat) Save

func (s *ReadStat) Save() (map[string]bigquery.Value, string, error)

Save is bigquery.ValueSaver interface

type ReadStatsParam

type ReadStatsParam struct {
	Table string
}

type ReadStatsTopTable

type ReadStatsTopTable string
const (
	ReadStatsTopMinuteTable   ReadStatsTopTable = "spanner_sys.read_stats_top_minute"
	ReadStatsTop10MinuteTable ReadStatsTopTable = "spanner_sys.read_stats_top_10minute"
	ReadStatsTopHourTable     ReadStatsTopTable = "spanner_sys.read_stats_top_hour"
)

type Service

type Service struct {
	Spanner *spanner.Client
	BQ      *bigquery.Client
	// contains filtered or unexported fields
}

func NewService

func NewService(ctx context.Context, bq *bigquery.Client) (*Service, error)

NewService is Serviceを生成する

func NewServiceWithSpannerClient

func NewServiceWithSpannerClient(ctx context.Context, bq *bigquery.Client, spannerClient *spanner.Client) (*Service, error)

NewServiceWithSpannerClient is Statsを取得したいSpanner DBが1つしかないのであれば、Spanner Clientを設定して、Serviceを作成する

func (*Service) Close

func (s *Service) Close() error

func (*Service) CopyLockStats added in v1.11.0

func (s *Service) CopyLockStats(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, lockStatsTable LockStatsTopTable, intervalEnd time.Time) (int, error)

CopyLockStats is SpannerからLock Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyLockStatsWithSpannerClient added in v1.11.0

func (s *Service) CopyLockStatsWithSpannerClient(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, lockStatsTable LockStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (insertCount int, err error)

CopyLockStatsWithSpannerClient is SpannerからLock Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyQueryStats

func (s *Service) CopyQueryStats(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, queryStatsTable QueryStatsTopTable, intervalEnd time.Time) (int, error)

CopyQueryStats is SpannerからQuery Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyQueryStatsWithSpannerClient

func (s *Service) CopyQueryStatsWithSpannerClient(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, queryStatsTable QueryStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (insertCount int, err error)

CopyQueryStatsWithSpannerClient is SpannerからQuery Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyReadStats

func (s *Service) CopyReadStats(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, readStatsTable ReadStatsTopTable, intervalEnd time.Time) (int, error)

CopyQueryStats is SpannerからQuery Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyReadStatsWithSpannerClient

func (s *Service) CopyReadStatsWithSpannerClient(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, readStatsTable ReadStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (insertCount int, err error)

CopyReadStatsWithSpannerClient is SpannerからRead Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyTxStats

func (s *Service) CopyTxStats(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, txStatsTable TxStatsTopTable, intervalEnd time.Time) (int, error)

CopyTxStats is SpannerからTx Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CopyTxStatsWithSpannerClient

func (s *Service) CopyTxStatsWithSpannerClient(ctx context.Context, dataset *bigquery.Dataset, bigQueryTable string, txStatsTable TxStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (insertCount int, err error)

CopyTxStatsWithSpannerClient is SpannerからTx Statsを引っ張ってきて、BigQueryにCopyしていく

func (*Service) CreateLockStatsTable added in v1.11.0

func (s *Service) CreateLockStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) error

CreateLockStatsTable is LockStatsをCopyするTableをBigQueryに作成する

func (*Service) CreateQueryStatsTable

func (s *Service) CreateQueryStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) error

CreateQueryStatsTable is QueryStatsをCopyするTableをBigQueryに作成する

func (*Service) CreateReadStatsTable

func (s *Service) CreateReadStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) error

CreateReadStatsTable is ReadStatsをCopyするTableをBigQueryに作成する

func (*Service) CreateTxStatsTable

func (s *Service) CreateTxStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) error

CreateTxStatsTable is TxStatsをCopyするTableをBigQueryに作成する

func (*Service) GetLockStats added in v1.11.0

func (s *Service) GetLockStats(ctx context.Context, table TxStatsTopTable, intervalEnd time.Time) ([]*LockStat, error)

GetLockStats is SpannerからLockStatsを取得する

func (*Service) GetLockStatsWithSpannerClient added in v1.11.0

func (s *Service) GetLockStatsWithSpannerClient(ctx context.Context, table TxStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (stats []*LockStat, err error)

GetLockStatsWithSpannerClient is 指定したSpannerClientを利用して、SpannerからLockStatsを取得する

func (*Service) GetQueryStats

func (s *Service) GetQueryStats(ctx context.Context, table QueryStatsTopTable, intervalEnd time.Time) ([]*QueryStat, error)

GetQueryStats is SpannerからQueryStatsを取得する

func (*Service) GetQueryStatsWithSpannerClient

func (s *Service) GetQueryStatsWithSpannerClient(ctx context.Context, table QueryStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (stats []*QueryStat, err error)

GetQueryStatsWithSpannerClient is 指定したSpannerClientを利用して、SpannerからQueryStatsを取得する

func (*Service) GetReadStats

func (s *Service) GetReadStats(ctx context.Context, table ReadStatsTopTable, intervalEnd time.Time) ([]*ReadStat, error)

GetReadStats is SpannerからTxStatsを取得する

func (*Service) GetReadStatsWithSpannerClient

func (s *Service) GetReadStatsWithSpannerClient(ctx context.Context, table ReadStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (stats []*ReadStat, err error)

GetReadStatsWithSpannerClient is 指定したSpannerClientを利用して、SpannerからQueryStatsを取得する

func (*Service) GetTxStats

func (s *Service) GetTxStats(ctx context.Context, table TxStatsTopTable, intervalEnd time.Time) ([]*TxStat, error)

GetTxStats is SpannerからTxStatsを取得する

func (*Service) GetTxStatsWithSpannerClient

func (s *Service) GetTxStatsWithSpannerClient(ctx context.Context, table TxStatsTopTable, spannerClient *spanner.Client, intervalEnd time.Time) (stats []*TxStat, err error)

GetTxStatsWithSpannerClient is 指定したSpannerClientを利用して、SpannerからTxStatsを取得する

func (*Service) UpdateLockStatsTable added in v1.13.0

func (s *Service) UpdateLockStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) (*bigquery.TableMetadata, error)

UpdateLockStatsTable is BigQuery上にあるTxStats TableのSchemaをUpdateする 途中でColumnが追加された時に使う

func (*Service) UpdateQueryStatsTable added in v1.7.0

func (s *Service) UpdateQueryStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) (*bigquery.TableMetadata, error)

UpdateQueryStatsTable is BigQuery上にあるQueryStats TableのSchemaをUpdateする 途中でColumnが追加されたときに使う

func (*Service) UpdateReadStatsTable added in v1.13.0

func (s *Service) UpdateReadStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) (*bigquery.TableMetadata, error)

UpdateReadStatsTable is BigQuery上にあるReadStats TableのSchemaをUpdateする 途中でColumnが追加された時に使う

func (*Service) UpdateTxStatsTable added in v1.13.0

func (s *Service) UpdateTxStatsTable(ctx context.Context, dataset *bigquery.Dataset, table string) (*bigquery.TableMetadata, error)

UpdateTxStatsTable is BigQuery上にあるTxStats TableのSchemaをUpdateする 途中でColumnが追加された時に使う

type TxStat added in v1.11.0

type TxStat struct {
	IntervalEnd              time.Time `spanner:"interval_end"` // End of the time interval that the included query executions occurred in.
	Fprint                   int64     // Fingerprint is the hash calculated based on the operations involved in the transaction. INTERVAL_END and FPRINT together act as an unique key for these tables.
	ReadColumns              []string  `spanner:"read_columns"`               // The set of columns that were read by the transaction.
	WriteConstructiveColumns []string  `spanner:"write_constructive_columns"` // The set of columns that were constructively written (i.e. assigned to new values) by the transaction.
	WriteDeleteTables        []string  `spanner:"write_delete_tables"`        // The set of tables that had rows deleted or replaced by the transaction.
	CommitAttemptCount       int64     `spanner:"commit_attempt_count"`       // 	Total number of commit attempts on the transaction.
	CommitAbortCount         int64     `spanner:"commit_abort_count"`         // Number of times the commits were aborted for the transaction.

	/*
	  Number of commit attempts that are retries from previously aborted attempts.
	  A Cloud Spanner transaction may have been tried multiple times before it commits due to lock contentions or transient events.
	  A high number of retries relative to commit attempts indicates that there may be issues worth investigating.
	  For more information, see Understanding transactions and commit counts on this page.
	  https://cloud.google.com/spanner/docs/introspection/transaction-statistics?hl=en#commit-counts
	*/
	CommitRetryCount int64 `spanner:"commit_retry_count"`

	CommitFailedPreconditionCount int64 `spanner:"commit_failed_precondition_count"` // Total number of precondition failures (FAILED_PRECONDITION) for the transaction.

	AvgParticipants         float64 `spanner:"avg_participants"`           // Average number of participants in each commit attempt. To learn more about participants, see Life of Cloud Spanner Reads & Writes.
	AvgTotalLatencySeconds  float64 `spanner:"avg_total_latency_seconds"`  // Average seconds taken from the first operation of the transaction to commit/abort.
	AvgCommitLatencySeconds float64 `spanner:"avg_commit_latency_seconds"` // Average seconds taken to perform the commit operation.
	AvgBytes                float64 `spanner:"avg_bytes"`                  // Average number of bytes written by the transaction.
}

func (*TxStat) InsertID added in v1.11.0

func (s *TxStat) InsertID() (string, error)

InsertID is 同じデータをBigQueryになるべく入れないようにデータからInsertIDを作成する

func (*TxStat) Save added in v1.11.0

func (s *TxStat) Save() (map[string]bigquery.Value, string, error)

Save is bigquery.ValueSaver interface

type TxStatsParam

type TxStatsParam struct {
	Table string
}

type TxStatsTopTable

type TxStatsTopTable string
const (
	TxStatsTopMinuteTable   TxStatsTopTable = "spanner_sys.txn_stats_top_minute"
	TxStatsTop10MinuteTable TxStatsTopTable = "spanner_sys.txn_stats_top_10minute"
	TxStatsTopHourTable     TxStatsTopTable = "spanner_sys.txn_stats_top_hour"
)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL