Documentation ¶
Overview ¶
Package pilosa implements the core of the Pilosa distributed bitmap index. It contains all the domain objects, interfaces, and logic that defines pilosa.
Index ¶
- Constants
- Variables
- func CompareTransactions(t1, t2 *Transaction) error
- func DecodeAttrs(v []byte) (map[string]interface{}, error)
- func EncodeAttrs(attr map[string]interface{}) ([]byte, error)
- func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64
- func MarshalInternalMessage(m Message, s Serializer) ([]byte, error)
- func NewRankCache(maxEntries uint32) *rankCache
- func OptAPIImportWorkerPoolSize(size int) apiOption
- func OptAPIServer(s *Server) apiOption
- func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap
- func VersionInfo() string
- func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap
- type API
- func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error)
- func (api *API) ApplySchema(ctx context.Context, s *Schema, remote bool) error
- func (api *API) AvailableShardsByIndex(ctx context.Context) map[string]*roaring.Bitmap
- func (api *API) Close() error
- func (api *API) ClusterMessage(ctx context.Context, reqBody io.Reader) error
- func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error)
- func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error)
- func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName string, shardID uint64) error
- func (api *API) DeleteField(ctx context.Context, indexName string, fieldName string) error
- func (api *API) DeleteIndex(ctx context.Context, indexName string) error
- func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error
- func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, ...) error
- func (api *API) Field(ctx context.Context, indexName, fieldName string) (*Field, error)
- func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error)
- func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error)
- func (api *API) FragmentBlockData(ctx context.Context, body io.Reader) ([]byte, error)
- func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error)
- func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error)
- func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Transaction, error)
- func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error)
- func (api *API) Hosts(ctx context.Context) []*Node
- func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error
- func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error
- func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, ...) (err error)
- func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error
- func (api *API) Index(ctx context.Context, indexName string) (*Index, error)
- func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error)
- func (api *API) Info() serverInfo
- func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error)
- func (api *API) LongQueryTime() time.Duration
- func (api *API) MaxShards(ctx context.Context) map[string]uint64
- func (api *API) Node() *Node
- func (api *API) PrimaryReplicaNodeURL() url.URL
- func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error)
- func (api *API) RecalculateCaches(ctx context.Context) error
- func (api *API) RemoveNode(id string) (*Node, error)
- func (api *API) ResizeAbort() error
- func (api *API) Schema(ctx context.Context) []*IndexInfo
- func (api *API) SetCoordinator(ctx context.Context, id string) (oldNode, newNode *Node, err error)
- func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error)
- func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, ...) (*Transaction, error)
- func (api *API) State() string
- func (api *API) StatsWithTags(tags []string) stats.StatsClient
- func (api *API) Transactions(ctx context.Context) (map[string]*Transaction, error)
- func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error)
- func (api *API) TranslateIDs(ctx context.Context, r io.Reader) (_ []byte, err error)
- func (api *API) TranslateIndexIDs(ctx context.Context, indexName string, ids []uint64) ([]string, error)
- func (api *API) TranslateIndexKey(ctx context.Context, indexName string, key string, writable bool) (uint64, error)
- func (api *API) TranslateKeys(ctx context.Context, r io.Reader) (_ []byte, err error)
- func (api *API) Version() string
- func (api *API) Views(ctx context.Context, indexName string, fieldName string) ([]*view, error)
- type ActiveQueryStatus
- type AttrBlock
- type AttrStore
- type BadRequestError
- type Bit
- type BlockDataRequest
- type BlockDataResponse
- type ClusterStatus
- type CmdIO
- type ColumnAttrSet
- type ConflictError
- type CreateFieldMessage
- type CreateIndexMessage
- type CreateShardMessage
- type CreateViewMessage
- type DeleteAvailableShardMessage
- type DeleteFieldMessage
- type DeleteIndexMessage
- type DeleteViewMessage
- type Error
- type Field
- func (f *Field) AddRemoteAvailableShards(b *roaring.Bitmap) error
- func (f *Field) AvailableShards() *roaring.Bitmap
- func (f *Field) CacheSize() uint32
- func (f *Field) ClearBit(rowID, colID uint64) (changed bool, err error)
- func (f *Field) ClearValue(columnID uint64) (changed bool, err error)
- func (f *Field) Close() error
- func (f *Field) CreatedAt() int64
- func (f *Field) ForeignIndex() string
- func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) error
- func (f *Field) Index() string
- func (f *Field) Keys() bool
- func (f *Field) MaxForShard(shard uint64, filter *Row) (ValCount, error)
- func (f *Field) MinForShard(shard uint64, filter *Row) (ValCount, error)
- func (f *Field) Name() string
- func (f *Field) Open() error
- func (f *Field) Options() FieldOptions
- func (f *Field) Path() string
- func (f *Field) Range(name string, op pql.Token, predicate int64) (*Row, error)
- func (f *Field) RemoveAvailableShard(v uint64) error
- func (f *Field) Row(rowID uint64) (*Row, error)
- func (f *Field) RowAttrStore() AttrStore
- func (f *Field) RowTime(rowID uint64, time time.Time, quantum string) (*Row, error)
- func (f *Field) SetBit(rowID, colID uint64, t *time.Time) (changed bool, err error)
- func (f *Field) SetCacheSize(v uint32) error
- func (f *Field) SetValue(columnID uint64, value int64) (changed bool, err error)
- func (f *Field) StringValue(columnID uint64) (value string, exists bool, err error)
- func (f *Field) TimeQuantum() TimeQuantum
- func (f *Field) TranslateStore() TranslateStore
- func (f *Field) TranslateStorePath() string
- func (f *Field) Type() string
- func (f *Field) Value(columnID uint64) (value int64, exists bool, err error)
- type FieldInfo
- type FieldOption
- func OptFieldForeignIndex(index string) FieldOption
- func OptFieldKeys() FieldOption
- func OptFieldTypeBool() FieldOption
- func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption
- func OptFieldTypeDefault() FieldOption
- func OptFieldTypeInt(min, max int64) FieldOption
- func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption
- func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption
- func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption
- type FieldOptions
- type FieldRow
- type FieldStatus
- type FieldValue
- type FragmentBlock
- type FragmentInfo
- type GCNotifier
- type GroupCount
- type GroupCounts
- type Handler
- type Hasher
- type Holder
- func (h *Holder) Activate()
- func (h *Holder) Close() error
- func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error)
- func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error)
- func (h *Holder) DeleteIndex(name string) error
- func (h *Holder) Field(index, name string) *Field
- func (h *Holder) FinishTransaction(ctx context.Context, id string) (*Transaction, error)
- func (h *Holder) GetTransaction(ctx context.Context, id string) (*Transaction, error)
- func (h *Holder) HasData() (bool, error)
- func (h *Holder) Index(name string) *Index
- func (h *Holder) IndexPath(name string) string
- func (h *Holder) Indexes() []*Index
- func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error)
- func (h *Holder) Open() error
- func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error)
- func (h *Holder) Schema() []*IndexInfo
- func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
- func (h *Holder) Transactions(ctx context.Context) (map[string]*Transaction, error)
- type HolderFilter
- type HolderFilterAll
- type HolderFilterParams
- type HolderInfo
- type HolderOperator
- type HolderOpts
- type HolderProcess
- type HolderProcessNone
- type ImportColumnAttrsRequest
- type ImportOption
- type ImportOptions
- type ImportRequest
- type ImportResponse
- type ImportRoaringRequest
- type ImportValueRequest
- type InMemTransactionStore
- type InMemTranslateStore
- func (s *InMemTranslateStore) Close() error
- func (s *InMemTranslateStore) EntryReader(ctx context.Context, offset uint64) (TranslateEntryReader, error)
- func (s *InMemTranslateStore) ForceSet(id uint64, key string) error
- func (s *InMemTranslateStore) MaxID() (uint64, error)
- func (s *InMemTranslateStore) PartitionID() int
- func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error)
- func (s *InMemTranslateStore) ReadOnly() bool
- func (s *InMemTranslateStore) SetReadOnly(v bool)
- func (s *InMemTranslateStore) TranslateID(id uint64) (string, error)
- func (s *InMemTranslateStore) TranslateIDs(ids []uint64) ([]string, error)
- func (s *InMemTranslateStore) TranslateKey(key string, writable bool) (uint64, error)
- func (s *InMemTranslateStore) TranslateKeys(keys []string, writable bool) (_ []uint64, err error)
- func (s *InMemTranslateStore) WriteNotify() <-chan struct{}
- func (s *InMemTranslateStore) WriteTo(w io.Writer) (int64, error)
- type Index
- func (i *Index) AvailableShards() *roaring.Bitmap
- func (i *Index) Close() error
- func (i *Index) ColumnAttrStore() AttrStore
- func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error)
- func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error)
- func (i *Index) CreatedAt() int64
- func (i *Index) DeleteField(name string) error
- func (i *Index) Field(name string) *Field
- func (i *Index) Fields() []*Field
- func (i *Index) Keys() bool
- func (i *Index) Name() string
- func (i *Index) Open() error
- func (i *Index) OpenWithTimestamp() error
- func (i *Index) Options() IndexOptions
- func (i *Index) Path() string
- func (i *Index) TranslateStore(partitionID int) TranslateStore
- func (i *Index) TranslateStorePath(partitionID int) string
- type IndexInfo
- type IndexOptions
- type IndexStatus
- type IndexTranslateOffsetMap
- type InspectRequest
- type InspectRequestParams
- type InspectResponse
- type InternalClient
- type InternalQueryClient
- type Message
- type MessageProcessingError
- type MultiTranslateEntryReader
- type Node
- type NodeEvent
- type NodeEventType
- type NodeStateMessage
- type NodeStatus
- type Nodes
- func (a Nodes) Clone() []*Node
- func (a Nodes) Contains(n *Node) bool
- func (a Nodes) ContainsID(id string) bool
- func (a Nodes) Filter(n *Node) []*Node
- func (a Nodes) FilterID(id string) []*Node
- func (a Nodes) FilterURI(uri URI) []*Node
- func (a Nodes) IDs() []string
- func (a Nodes) NodeByID(id string) *Node
- func (a Nodes) URIs() []URI
- type NotFoundError
- type OpenTransactionStoreFunc
- type OpenTranslateReaderFunc
- type OpenTranslateStoreFunc
- type Pair
- type PairField
- type Pairs
- type PairsField
- type PreconditionFailedError
- type QueryRequest
- type QueryResponse
- type RecalculateCaches
- type ResizeInstruction
- type ResizeInstructionComplete
- type ResizeSource
- type Row
- func (r *Row) Any() bool
- func (r *Row) Columns() []uint64
- func (r *Row) Count() uint64
- func (r *Row) Difference(others ...*Row) *Row
- func (r *Row) Freeze()
- func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row
- func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64
- func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row
- func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row
- func (r *Row) Includes(col uint64) bool
- func (r *Row) Intersect(other *Row) *Row
- func (r *Row) IsEmpty() bool
- func (r *Row) MarshalJSON() ([]byte, error)
- func (r *Row) Merge(other *Row)
- func (r *Row) Roaring() []byte
- func (r *Row) Segments() []rowSegment
- func (r *Row) SetBit(i uint64) (changed bool)
- func (r *Row) Shift(n int64) (*Row, error)
- func (r *Row) ToRows(callback func(*pb.RowResponse) error) error
- func (r *Row) ToTable() (*pb.TableResponse, error)
- func (r *Row) Union(others ...*Row) *Row
- func (r *Row) Xor(other *Row) *Row
- type RowIDs
- type RowIdentifiers
- type Schema
- type Serializer
- type Server
- func (s *Server) Close() error
- func (srv *Server) FinishTransaction(ctx context.Context, id string, remote bool) (*Transaction, error)
- func (srv *Server) GetTransaction(ctx context.Context, id string, remote bool) (*Transaction, error)
- func (s *Server) Holder() *Holder
- func (s *Server) InternalClient() InternalClient
- func (s *Server) NodeID() string
- func (s *Server) Open() error
- func (s *Server) SendAsync(m Message) error
- func (s *Server) SendSync(m Message) error
- func (s *Server) SendTo(to *Node, m Message) error
- func (srv *Server) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool, ...) (*Transaction, error)
- func (s *Server) SyncData() error
- func (srv *Server) Transactions(ctx context.Context) (map[string]*Transaction, error)
- func (s *Server) UpAndDown() error
- type ServerOption
- func OptServerAntiEntropyInterval(interval time.Duration) ServerOption
- func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption
- func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption
- func OptServerClusterHasher(h Hasher) ServerOption
- func OptServerDataDir(dir string) ServerOption
- func OptServerDiagnosticsInterval(dur time.Duration) ServerOption
- func OptServerExecutorPoolSize(size int) ServerOption
- func OptServerGCNotifier(gcn GCNotifier) ServerOption
- func OptServerGRPCURI(uri *URI) ServerOption
- func OptServerInternalClient(c InternalClient) ServerOption
- func OptServerIsCoordinator(is bool) ServerOption
- func OptServerLogger(l logger.Logger) ServerOption
- func OptServerLongQueryTime(dur time.Duration) ServerOption
- func OptServerMaxWritesPerRequest(n int) ServerOption
- func OptServerMetricInterval(dur time.Duration) ServerOption
- func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption
- func OptServerNodeID(nodeID string) ServerOption
- func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption
- func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption
- func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption
- func OptServerReplicaN(n int) ServerOption
- func OptServerSerializer(ser Serializer) ServerOption
- func OptServerStatsClient(sc stats.StatsClient) ServerOption
- func OptServerSystemInfo(si SystemInfo) ServerOption
- func OptServerURI(uri *URI) ServerOption
- type SetCoordinatorMessage
- type SignedRow
- type SnapshotQueue
- type SystemInfo
- type TimeQuantum
- type Topology
- type Transaction
- type TransactionManager
- func (tm *TransactionManager) Finish(ctx context.Context, id string) (*Transaction, error)
- func (tm *TransactionManager) Get(ctx context.Context, id string) (*Transaction, error)
- func (tm *TransactionManager) List(ctx context.Context) (map[string]*Transaction, error)
- func (tm *TransactionManager) ResetDeadline(ctx context.Context, id string) (*Transaction, error)
- func (tm *TransactionManager) Start(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
- type TransactionMessage
- type TransactionStats
- type TransactionStore
- type TranslateEntry
- type TranslateEntryReader
- type TranslateIDsRequest
- type TranslateIDsResponse
- type TranslateKeysRequest
- type TranslateKeysResponse
- type TranslateOffsetMap
- func (m TranslateOffsetMap) FieldOffset(index, name string) uint64
- func (m TranslateOffsetMap) IndexPartitionOffset(name string, partitionID int) uint64
- func (m TranslateOffsetMap) SetFieldOffset(index, name string, offset uint64)
- func (m TranslateOffsetMap) SetIndexPartitionOffset(name string, partitionID int, offset uint64)
- type TranslateStore
- type TranslationResizeSource
- type URI
- func (u *URI) HostPort() string
- func (u *URI) MarshalJSON() ([]byte, error)
- func (u *URI) Path(path string) string
- func (u *URI) Set(value string) error
- func (u *URI) SetPort(port uint16)
- func (u URI) String() string
- func (u URI) Type() string
- func (u *URI) URL() url.URL
- func (u *URI) UnmarshalJSON(b []byte) error
- type URIs
- type UpdateCoordinatorMessage
- type ValCount
- type ViewInfo
Constants ¶
const ( // DefaultPartitionN is the default number of partitions in a cluster. DefaultPartitionN = 256 // ClusterState represents the state returned in the /status endpoint. ClusterStateStarting = "STARTING" ClusterStateDegraded = "DEGRADED" // cluster is running but we've lost some # of hosts >0 but < replicaN ClusterStateNormal = "NORMAL" ClusterStateResizing = "RESIZING" )
const ( TRANSACTION_START = "start" TRANSACTION_FINISH = "finish" TRANSACTION_VALIDATE = "validate" )
Transaction Actions
const ( DefaultFieldType = FieldTypeSet DefaultCacheType = CacheTypeRanked // Default ranked field cache DefaultCacheSize = 50000 )
Default field settings.
const ( FieldTypeSet = "set" FieldTypeInt = "int" FieldTypeTime = "time" FieldTypeMutex = "mutex" FieldTypeBool = "bool" FieldTypeDecimal = "decimal" )
Field types.
const ( CacheTypeLRU = "lru" CacheTypeRanked = "ranked" CacheTypeNone = "none" )
Cache types.
const ( // ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16. // shardWidthExponent = 20 // set in shardwidthNN.go files ShardWidth = 1 << shardwidth.Exponent // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 )
const ( RequestActionSet = "set" RequestActionClear = "clear" RequestActionOverwrite = "overwrite" )
const ( MetricCreateIndex = "create_index_total" MetricDeleteIndex = "delete_index_total" MetricCreateField = "create_field_total" MetricDeleteField = "delete_field_total" MetricDeleteAvailableShard = "delete_available_shard_total" MetricRecalculateCache = "recalculate_cache_total" MetricInvalidateCache = "invalidate_cache_total" MetricRankCacheLength = "rank_cache_length" MetricCacheThresholdReached = "cache_threshold_reached_total" MetricRow = "query_row_total" MetricRowBSI = "query_row_bsi_total" MetricSetRowAttrs = "query_setrowattrs_total" MetricSetColumnAttrs = "query_setcolumnattrs_total" MetricSetBit = "set_bit_total" MetricClearBit = "clear_bit_total" MetricImportingN = "importing_total" MetricImportedN = "imported_total" MetricClearingN = "clearing_total" MetricClearedN = "cleared_total" MetricSnapshotDurationSeconds = "snapshot_duration_seconds" MetricBlockRepair = "block_repair_total" MetricSyncFieldDurationSeconds = "sync_field_duration_seconds" MetricSyncIndexDurationSeconds = "sync_index_duration_seconds" MetricColumnAttrStoreBlocks = "column_attr_store_blocks_total" MetricColumnAttrDiff = "column_attr_diff_total" MetricRowAttrStoreBlocks = "row_attr_store_blocks_total" MetricRowAttrDiff = "row_attr_diff_total" MetricHTTPRequest = "http_request_duration_seconds" MetricGRPCUnaryQueryDurationSeconds = "grpc_request_pql_unary_query_duration_seconds" MetricGRPCUnaryFormatDurationSeconds = "grpc_request_pql_unary_format_duration_seconds" MetricGRPCStreamQueryDurationSeconds = "grpc_request_pql_stream_query_duration_seconds" MetricGRPCStreamFormatDurationSeconds = "grpc_request_pql_stream_format_duration_seconds" MetricMaxShard = "maximum_shard" MetricAntiEntropy = "antientropy_total" MetricAntiEntropyDurationSeconds = "antientropy_duration_seconds" MetricGarbageCollection = "garbage_collection_total" MetricGoroutines = "goroutines" MetricOpenFiles = "open_files" MetricHeapAlloc = "heap_alloc" MetricHeapInuse = "heap_inuse" MetricStackInuse = "stack_inuse" MetricMallocs = "mallocs" MetricFrees = "frees" MetricTransactionStart = "transaction_start" MetricTransactionEnd = "trasaction_end" MetricTransactionBlocked = "transaction_blocked" MetricExclusiveTransactionRequest = "transaction_exclusive_request" MetricExclusiveTransactionActive = "transaction_exclusive_active" MetricExclusiveTransactionEnd = "trasaction_exclusive_end" MetricExclusiveTransactionBlocked = "transaction_exclusive_blocked" )
const ErrTransactionExclusive = Error("there is an exclusive transaction, try later")
const ErrTransactionExists = Error("transaction with the given id already exists")
const ErrTransactionNotFound = Error("transaction not found")
const TimeFormat = "2006-01-02T15:04"
TimeFormat is the go-style time format used to parse string dates.
Variables ¶
var ( ErrHostRequired = errors.New("host required") ErrIndexRequired = errors.New("index required") ErrIndexExists = errors.New("index already exists") ErrIndexNotFound = errors.New("index not found") ErrForeignIndexNotFound = errors.New("foreign index not found") // ErrFieldRequired is returned when no field is specified. ErrFieldRequired = errors.New("field required") ErrColumnRequired = errors.New("column required") ErrFieldExists = errors.New("field already exists") ErrFieldNotFound = errors.New("field not found") ErrBSIGroupNotFound = errors.New("bsigroup not found") ErrBSIGroupExists = errors.New("bsigroup already exists") ErrBSIGroupNameRequired = errors.New("bsigroup name required") ErrInvalidBSIGroupType = errors.New("invalid bsigroup type") ErrInvalidBSIGroupRange = errors.New("invalid bsigroup range") ErrInvalidBSIGroupValueType = errors.New("invalid bsigroup value type") ErrBSIGroupValueTooLow = errors.New("bsigroup value too low") ErrBSIGroupValueTooHigh = errors.New("bsigroup value too high") ErrInvalidRangeOperation = errors.New("invalid range operation") ErrInvalidBetweenValue = errors.New("invalid value for between operation") ErrDecimalOutOfRange = errors.New("decimal value out of range") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") ErrName = errors.New("invalid index or field name, must match [a-z][a-z0-9_-]* and contain at most 230 characters") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") ErrQueryCancelled = errors.New("query cancelled") ErrQueryTimeout = errors.New("query timeout") ErrTooManyWrites = errors.New("too many write commands") // TODO(2.0) poorly named - used when a *node* doesn't own a shard. Probably // we won't need this error at all by 2.0 though. ErrClusterDoesNotOwnShard = errors.New("node does not own shard") // ErrPreconditionFailed is returned when specified index/field createdAt timestamps don't match ErrPreconditionFailed = errors.New("precondition failed") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") ErrResizeNoReplicas = errors.New("not enough data to perform resize (replica factor may need to be increased)") ErrNotImplemented = errors.New("not implemented") ErrFieldsArgumentRequired = errors.New("fields argument required") ErrExpectedFieldListArgument = errors.New("expected field list argument") ErrIntFieldWithKeys = errors.New("int field cannot be created with 'keys=true' option") ErrDecimalFieldWithKeys = errors.New("decimal field cannot be created with 'keys=true' option") )
System errors.
var ( ErrTranslateStoreClosed = errors.New("translate store closed") ErrTranslateStoreReaderClosed = errors.New("translate store reader closed") ErrReplicationNotSupported = errors.New("replication not supported") ErrTranslateStoreReadOnly = errors.New("translate store could not find or create key, translate store read only") ErrTranslateStoreNotFound = errors.New("translate store not found") ErrTranslatingKeyNotFound = errors.New("translating key not found") ErrCannotOpenV1TranslateFile = errors.New("cannot open v1 translate .keys file") )
Translate store errors.
var BuildTime string
var Commit string
var ErrInvalidTimeQuantum = errors.New("invalid time quantum")
ErrInvalidTimeQuantum is returned when parsing a time quantum.
var NopBroadcaster broadcaster = &nopBroadcaster{}
NopBroadcaster represents a Broadcaster that doesn't do anything.
var NopTranslationSyncer translationSyncer = &nopTranslationSyncer{}
NopTranslationSyncer represents a translationSyncer that doesn't do anything.
var Variant string
var Version string
Functions ¶
func CompareTransactions ¶
func CompareTransactions(t1, t2 *Transaction) error
func DecodeAttrs ¶
DecodeAttrs decodes a byte slice into an attribute map.
func EncodeAttrs ¶
EncodeAttrs encodes an attribute map into a byte slice.
func GenerateNextPartitionedID ¶
GenerateNextPartitionedID returns the next ID within the same partition.
func MarshalInternalMessage ¶
func MarshalInternalMessage(m Message, s Serializer) ([]byte, error)
MarshalInternalMessage serializes the pilosa message and adds pilosa internal type info which is used by the internal messaging stuff.
func NewRankCache ¶
func NewRankCache(maxEntries uint32) *rankCache
NewRankCache returns a new instance of RankCache.
func OptAPIImportWorkerPoolSize ¶
func OptAPIImportWorkerPoolSize(size int) apiOption
func OptAPIServer ¶
func OptAPIServer(s *Server) apiOption
func UnwrapBitmap ¶
UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
func VersionInfo ¶
func VersionInfo() string
Types ¶
type API ¶
type API struct { Serializer Serializer // contains filtered or unexported fields }
API provides the top level programmatic interface to Pilosa. It is usually wrapped by a handler which provides an external interface (e.g. HTTP).
func (*API) ActiveQueries ¶
func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error)
func (*API) ApplySchema ¶
ApplySchema takes the given schema and applies it across the cluster (if remote is false), or just to this node (if remote is true). This is designed for the use case of replicating a schema from one Pilosa cluster to another which is initially empty. It is not officially supported in other scenarios and may produce surprising results.
func (*API) AvailableShardsByIndex ¶
AvailableShardsByIndex returns bitmaps of shards with available by index name.
func (*API) ClusterMessage ¶
ClusterMessage is for internal use. It decodes a protobuf message out of the body and forwards it to the BroadcastHandler.
func (*API) CreateField ¶
func (api *API) CreateField(ctx context.Context, indexName string, fieldName string, opts ...FieldOption) (*Field, error)
CreateField makes the named field in the named index with the given options. This method currently only takes a single functional option, but that may be changed in the future to support multiple options.
func (*API) CreateIndex ¶
func (api *API) CreateIndex(ctx context.Context, indexName string, options IndexOptions) (*Index, error)
CreateIndex makes a new Pilosa index.
func (*API) DeleteAvailableShard ¶
func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName string, shardID uint64) error
DeleteAvailableShard a shard ID from the available shard set cache.
func (*API) DeleteField ¶
DeleteField removes the named field from the named index. If the index is not found, an error is returned. If the field is not found, it is ignored and no action is taken.
func (*API) DeleteIndex ¶
DeleteIndex removes the named index. If the index is not found it does nothing and returns no error.
func (*API) DeleteView ¶
func (api *API) DeleteView(ctx context.Context, indexName string, fieldName string, viewName string) error
DeleteView removes the given view.
func (*API) ExportCSV ¶
func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName string, shard uint64, w io.Writer) error
ExportCSV encodes the fragment designated by the index,field,shard as CSV of the form <row>,<col>
func (*API) FieldAttrDiff ¶
func (api *API) FieldAttrDiff(ctx context.Context, indexName string, fieldName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error)
FieldAttrDiff determines the local row attribute data blocks which differ from those provided.
func (*API) FinishTransaction ¶
func (*API) FragmentBlockData ¶
FragmentBlockData is an endpoint for internal usage. It is not guaranteed to return anything useful. Currently it returns protobuf encoded row and column ids from a "block" which is a subdivision of a fragment.
func (*API) FragmentBlocks ¶
func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewName string, shard uint64) ([]FragmentBlock, error)
FragmentBlocks returns the checksums and block ids for all blocks in the specified fragment.
func (*API) FragmentData ¶
func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error)
FragmentData returns all data in the specified fragment.
func (*API) GetTransaction ¶
func (*API) GetTranslateEntryReader ¶
func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOffsetMap) (_ TranslateEntryReader, err error)
GetTranslateEntryReader provides an entry reader for key translation logs starting at offset.
func (*API) Hosts ¶
Hosts returns a list of the hosts in the cluster including their ID, URL, and which is the coordinator.
func (*API) Import ¶
func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOption) error
Import bulk imports data into a particular index,field,shard.
func (*API) ImportColumnAttrs ¶
func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsRequest, opts ...ImportOption) error
func (*API) ImportRoaring ¶
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error)
ImportRoaring is a low level interface for importing data to Pilosa when extremely high throughput is desired. The data must be encoded in a particular way which may be unintuitive (discussed below). The data is merged with existing data.
It takes as input a roaring bitmap which it uses as the data for the indicated index, field, and shard. The bitmap may be encoded according to the official roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec), or to the pilosa roaring spec which supports 64 bit integers (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format).
The data should be encoded the same way that Pilosa stores fragments internally. A bit "i" being set in the input bitmap indicates that the bit is set in Pilosa row "i/ShardWidth", and in column (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all of the rows in this shard of this field concatenated together in one long bitmap.
func (*API) ImportValue ¶
func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts ...ImportOption) error
ImportValue bulk imports values into a particular field.
func (*API) IndexAttrDiff ¶
func (api *API) IndexAttrDiff(ctx context.Context, indexName string, blocks []AttrBlock) (map[uint64]map[string]interface{}, error)
IndexAttrDiff determines the local column attribute data blocks which differ from those provided.
func (*API) Info ¶
func (api *API) Info() serverInfo
Info returns information about this server instance.
func (*API) Inspect ¶
func (api *API) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error)
func (*API) LongQueryTime ¶
LongQueryTime returns the configured threshold for logging/statting long running queries.
func (*API) MaxShards ¶
MaxShards returns the maximum shard number for each index in a map. TODO (2.0): This method has been deprecated. Instead, use AvailableShardsByIndex.
func (*API) PrimaryReplicaNodeURL ¶
PrimaryReplicaNodeURL returns the URL of the cluster's primary replica.
func (*API) Query ¶
func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, error)
Query parses a PQL query out of the request and executes it.
func (*API) RecalculateCaches ¶
RecalculateCaches forces all TopN caches to be updated. Used mainly for integration tests.
func (*API) RemoveNode ¶
RemoveNode puts the cluster into the "RESIZING" state and begins the job of removing the given node.
func (*API) ResizeAbort ¶
ResizeAbort stops the current resize job.
func (*API) Schema ¶
Schema returns information about each index in Pilosa including which fields they contain.
func (*API) SetCoordinator ¶
SetCoordinator makes a new Node the cluster coordinator.
func (*API) ShardNodes ¶
ShardNodes returns the node and all replicas which should contain a shard's data.
func (*API) StartTransaction ¶
func (*API) State ¶
State returns the cluster state which is usually "NORMAL", but could be "STARTING", "RESIZING", or potentially others. See cluster.go for more details.
func (*API) StatsWithTags ¶
func (api *API) StatsWithTags(tags []string) stats.StatsClient
StatsWithTags returns an instance of whatever implementation of StatsClient pilosa is using with the given tags.
func (*API) Transactions ¶
func (*API) TranslateData ¶
func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error)
TranslateData returns all translation data in the specified partition.
func (*API) TranslateIDs ¶
TranslateIDs handles a TranslateIDRequest.
func (*API) TranslateIndexIDs ¶
func (*API) TranslateIndexKey ¶
func (*API) TranslateKeys ¶
TranslateKeys handles a TranslateKeyRequest. ErrTranslatingKeyNotFound error will be swallowed here, so the empty response will be returned.
type ActiveQueryStatus ¶
type AttrStore ¶
type AttrStore interface { Path() string Open() error Close() error Attrs(id uint64) (m map[string]interface{}, err error) SetAttrs(id uint64, m map[string]interface{}) error SetBulkAttrs(m map[uint64]map[string]interface{}) error Blocks() ([]AttrBlock, error) BlockData(i uint64) (map[uint64]map[string]interface{}, error) }
AttrStore represents an interface for handling row/column attributes.
type BadRequestError ¶
type BadRequestError struct {
// contains filtered or unexported fields
}
BadRequestError wraps an error value to signify that a request could not be read, decoded, or parsed such that in an HTTP scenario, http.StatusBadRequest would be returned.
func NewBadRequestError ¶
func NewBadRequestError(err error) BadRequestError
NewBadRequestError returns err wrapped in a BadRequestError.
type Bit ¶
Bit represents the intersection of a row and a column. It can be specified by integer ids or string keys.
type BlockDataRequest ¶
BlockDataRequest describes the structure of a request for fragment block data.
type BlockDataResponse ¶
BlockDataResponse is the structured response of a block data request.
type ClusterStatus ¶
ClusterStatus describes the status of the cluster including its state and node topology.
type CmdIO ¶
type CmdIO struct { Stdin io.Reader Stdout io.Writer Stderr io.Writer // contains filtered or unexported fields }
CmdIO holds standard unix inputs and outputs.
type ColumnAttrSet ¶
type ColumnAttrSet struct { ID uint64 `json:"id"` Key string `json:"key,omitempty"` Attrs map[string]interface{} `json:"attrs,omitempty"` }
ColumnAttrSet represents a set of attributes for a vertical column in an index. Can have a set of attributes attached to it.
func (ColumnAttrSet) MarshalJSON ¶
func (cas ColumnAttrSet) MarshalJSON() ([]byte, error)
MarshalJSON marshals the ColumnAttrSet to JSON such that either a Key or an ID is included.
type ConflictError ¶
type ConflictError struct {
// contains filtered or unexported fields
}
ConflictError wraps an error value to signify that a conflict with an existing resource occurred such that in an HTTP scenario, http.StatusConflict would be returned.
type CreateFieldMessage ¶
type CreateFieldMessage struct { Index string Field string CreatedAt int64 Meta *FieldOptions }
CreateFieldMessage is an internal message indicating field creation.
type CreateIndexMessage ¶
type CreateIndexMessage struct { Index string CreatedAt int64 Meta *IndexOptions }
CreateIndexMessage is an internal message indicating index creation.
type CreateShardMessage ¶
CreateShardMessage is an internal message indicating shard creation.
type CreateViewMessage ¶
CreateViewMessage is an internal message indicating view creation.
type DeleteAvailableShardMessage ¶
DeleteAvailableShardMessage is an internal message indicating available shard deletion.
type DeleteFieldMessage ¶
DeleteFieldMessage is an internal message indicating field deletion.
type DeleteIndexMessage ¶
type DeleteIndexMessage struct {
Index string
}
DeleteIndexMessage is an internal message indicating index deletion.
type DeleteViewMessage ¶
DeleteViewMessage is an internal message indicating view deletion.
type Field ¶
type Field struct { Stats stats.StatsClient // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc // contains filtered or unexported fields }
Field represents a container for views.
func NewField ¶
func NewField(holder *Holder, path, index, name string, opts FieldOption) (*Field, error)
NewField returns a new instance of field. NOTE: This function is only used in tests, which is why it only takes a single `FieldOption` (the assumption being that it's of the type `OptFieldType*`). This means this function couldn't be used to set, for example, `FieldOptions.Keys`.
func (*Field) AddRemoteAvailableShards ¶
AddRemoteAvailableShards merges the set of available shards into the current known set and saves the set to a file.
func (*Field) AvailableShards ¶
AvailableShards returns a bitmap of shards that contain data.
func (*Field) ClearValue ¶
ClearValue removes a field value for a column.
func (*Field) ForeignIndex ¶
ForeignIndex returns the foreign index name attached to the field. Returns blank string if no foreign index exists.
func (*Field) Import ¶
func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts ...ImportOption) error
Import bulk imports data.
func (*Field) MaxForShard ¶
func (*Field) MinForShard ¶
MinForShard returns the minimum value which appears in this shard (this field must be an Int or Decimal field). It also returns the number of times the minimum value appears.
func (*Field) Options ¶
func (f *Field) Options() FieldOptions
Options returns all options for this field.
func (*Field) RemoveAvailableShard ¶
RemoveAvailableShard removes a shard from the bitmap cache.
NOTE: This can be overridden on the next sync so all nodes should be updated.
func (*Field) Row ¶
Row returns a row of the standard view. It seems this method is only being used by the test package, and the fact that it's only allowed on `set`,`mutex`, and `bool` fields is odd. This may be considered for deprecation in a future version.
func (*Field) RowAttrStore ¶
RowAttrStore returns the attribute storage.
func (*Field) RowTime ¶
RowTime gets the row at the particular time with the granularity specified by the quantum.
func (*Field) SetCacheSize ¶
SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. defaults to DefaultCacheSize 50000
func (*Field) StringValue ¶
StringValue reads an integer field value for a column, and converts it to a string based on a foreign index string key.
func (*Field) TimeQuantum ¶
func (f *Field) TimeQuantum() TimeQuantum
TimeQuantum returns the time quantum for the field.
func (*Field) TranslateStore ¶
func (f *Field) TranslateStore() TranslateStore
TranslateStore returns the field's translation store.
func (*Field) TranslateStorePath ¶
TranslateStorePath returns the translation database path for the field.
type FieldInfo ¶
type FieldInfo struct { Name string `json:"name"` CreatedAt int64 `json:"createdAt,omitempty"` Options FieldOptions `json:"options"` Views []*ViewInfo `json:"views,omitempty"` }
FieldInfo represents schema information for a field.
type FieldOption ¶
type FieldOption func(fo *FieldOptions) error
FieldOption is a functional option type for pilosa.fieldOptions.
func OptFieldForeignIndex ¶
func OptFieldForeignIndex(index string) FieldOption
OptFieldForeignIndex marks this field as a foreign key to another index. That is, the values of this field should be interpreted as referencing records (Pilosa columns) in another index. TODO explain where/how this is used by Pilosa.
func OptFieldKeys ¶
func OptFieldKeys() FieldOption
OptFieldKeys is a functional option on FieldOptions used to specify whether keys are used for this field.
func OptFieldTypeBool ¶
func OptFieldTypeBool() FieldOption
OptFieldTypeBool is a functional option on FieldOptions used to specify the field as being type `bool` and to provide any respective configuration values.
func OptFieldTypeDecimal ¶
func OptFieldTypeDecimal(scale int64, minmax ...pql.Decimal) FieldOption
OptFieldTypeDecimal is a functional option for creating a `decimal` field. Unless we decide to expand the range of supported values, `scale` is restricted to the range [0,19]. This supports anything from:
scale = 0: min: -9223372036854775808. max: 9223372036854775807.
to:
scale = 19: min: -0.9223372036854775808 max: 0.9223372036854775807
While it's possible to support scale values outside of this range, the coverage for those scales are no longer continuous. For example,
scale = -2: min : [-922337203685477580800, -100] GAPs: [-99, -1], [-199, -101] ... [-922337203685477580799, -922337203685477580701]
0
max : [100, 922337203685477580700] GAPs: [1, 99], [101, 199] ... [922337203685477580601, 922337203685477580699]
An alternative to this gap strategy would be to scale the supported range to a continuous 64-bit space (which is not unreasonable using bsiGroup.Base). The issue with this approach is that we would need to know which direction to favor. For example, there are two possible ranges for `scale = -2`:
min : [-922337203685477580800, -922337203685477580800+(2^64)] max : [922337203685477580700-(2^64), 922337203685477580700]
func OptFieldTypeDefault ¶
func OptFieldTypeDefault() FieldOption
OptFieldTypeDefault is a functional option on FieldOptions used to set the field type and cache setting to the default values.
func OptFieldTypeInt ¶
func OptFieldTypeInt(min, max int64) FieldOption
OptFieldTypeInt is a functional option on FieldOptions used to specify the field as being type `int` and to provide any respective configuration values.
func OptFieldTypeMutex ¶
func OptFieldTypeMutex(cacheType string, cacheSize uint32) FieldOption
OptFieldTypeMutex is a functional option on FieldOptions used to specify the field as being type `mutex` and to provide any respective configuration values.
func OptFieldTypeSet ¶
func OptFieldTypeSet(cacheType string, cacheSize uint32) FieldOption
OptFieldTypeSet is a functional option on FieldOptions used to specify the field as being type `set` and to provide any respective configuration values.
func OptFieldTypeTime ¶
func OptFieldTypeTime(timeQuantum TimeQuantum, opt ...bool) FieldOption
OptFieldTypeTime is a functional option on FieldOptions used to specify the field as being type `time` and to provide any respective configuration values. Pass true to skip creation of the standard view.
type FieldOptions ¶
type FieldOptions struct { Base int64 `json:"base,omitempty"` BitDepth uint `json:"bitDepth,omitempty"` Min pql.Decimal `json:"min,omitempty"` Max pql.Decimal `json:"max,omitempty"` Scale int64 `json:"scale,omitempty"` Keys bool `json:"keys"` NoStandardView bool `json:"noStandardView,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` CacheType string `json:"cacheType,omitempty"` Type string `json:"type,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` ForeignIndex string `json:"foreignIndex"` }
FieldOptions represents options to set when initializing a field.
func (*FieldOptions) MarshalJSON ¶
func (o *FieldOptions) MarshalJSON() ([]byte, error)
MarshalJSON marshals FieldOptions to JSON such that only those attributes associated to the field type are included.
type FieldRow ¶
type FieldRow struct { Field string `json:"field"` RowID uint64 `json:"rowID"` RowKey string `json:"rowKey,omitempty"` Value *int64 `json:"value,omitempty"` }
FieldRow is used to distinguish rows in a group by result.
func (FieldRow) MarshalJSON ¶
MarshalJSON marshals FieldRow to JSON such that either a Key or an ID is included.
type FieldStatus ¶
FieldStatus is an internal message representing the contents of a field.
type FieldValue ¶
FieldValue represents the value for a column within a range-encoded field.
type FragmentBlock ¶
FragmentBlock represents info about a subsection of the rows in a block. This is used for comparing data in remote blocks for active anti-entropy.
type FragmentInfo ¶
type FragmentInfo struct { BitmapInfo roaring.BitmapInfo BlockChecksums []FragmentBlock `json:"BlockChecksums,omitempty"` }
type GCNotifier ¶
type GCNotifier interface { Close() AfterGC() <-chan struct{} }
GCNotifier represents an interface for garbage collection notificationss.
var NopGCNotifier GCNotifier = &nopGCNotifier{}
NopGCNotifier represents a GCNotifier that doesn't do anything.
type GroupCount ¶
type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` Sum int64 `json:"sum"` }
GroupCount represents a result item for a group by query.
func (GroupCount) Compare ¶
func (g GroupCount) Compare(o GroupCount) int
Compare is used in ordering two GroupCount objects.
type GroupCounts ¶
type GroupCounts []GroupCount
GroupCounts is a list of GroupCount.
func (GroupCounts) ToRows ¶
func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error
ToRows implements the ToRowser interface.
func (GroupCounts) ToTable ¶
func (g GroupCounts) ToTable() (*pb.TableResponse, error)
ToTable implements the ToTabler interface.
type Handler ¶
Handler is the interface for the data handler, a wrapper around Pilosa's data store.
var NopHandler Handler = nopHandler{}
NopHandler is a no-op implementation of the Handler interface.
type Hasher ¶
type Hasher interface { // Hashes the key into a number between [0,N). Hash(key uint64, n int) int }
Hasher represents an interface to hash integers into buckets.
type Holder ¶
type Holder struct { NewAttrStore func(string) AttrStore // Stats Stats stats.StatsClient // Data directory path. Path string Logger logger.Logger SnapshotQueue SnapshotQueue // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc OpenTranslateReader OpenTranslateReaderFunc // Func to open whatever implementation of transaction store we're using. OpenTransactionStore OpenTransactionStoreFunc Opts HolderOpts // contains filtered or unexported fields }
Holder represents a container for indexes.
func (*Holder) Activate ¶
func (h *Holder) Activate()
Activate runs the background tasks relevant to keeping a holder in a stable state, such as scanning it for needed snapshots, or flushing caches. This is separate from opening because, while a server would nearly always want to do this, other use cases (like consistency checks of a data directory) need to avoid it even getting started.
func (*Holder) CreateIndex ¶
func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error)
CreateIndex creates an index. An error is returned if the index already exists.
func (*Holder) CreateIndexIfNotExists ¶
func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error)
CreateIndexIfNotExists returns an index by name. The index is created if it does not already exist.
func (*Holder) DeleteIndex ¶
DeleteIndex removes an index from the holder.
func (*Holder) FinishTransaction ¶
func (*Holder) GetTransaction ¶
func (*Holder) HasData ¶
HasData returns true if Holder contains at least one index. This is used to determine if the rebalancing of data is necessary when a node joins the cluster.
func (*Holder) Inspect ¶
func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, error)
func (*Holder) Process ¶
func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error)
Process loops through a holder based on the Check functions in op, calling the Process functions in op when indicated.
func (*Holder) StartTransaction ¶
func (*Holder) Transactions ¶
type HolderFilter ¶
type HolderFilter interface { CheckIndex(iname string) (process bool, recurse bool) CheckField(iname, fname string) (process bool, recurse bool) CheckView(iname, fname, vname string) (process bool, recurse bool) CheckFragment(iname, fname, vname string, shard uint64) (process bool) }
HolderFilter represents something that potentially filters out parts of a holder, indicating whether or not to process them, or recurse into them. It is permissible to recurse a thing without processing it, or process it without recursing it. For instance, something looking to accumulate statistics about views might return (true, false) from CheckView, while a fragment scanning operation would return (false, true) from everything above CheckFrag.
func NewHolderFilter ¶
func NewHolderFilter(params HolderFilterParams) (result HolderFilter, err error)
type HolderFilterAll ¶
type HolderFilterAll struct{}
HolderFilterAll is a placeholder type which always returns true for the check functions. You can embed it to make a HolderOperator which processes everything.
func (HolderFilterAll) CheckField ¶
func (HolderFilterAll) CheckField(string, string) (bool, bool)
func (HolderFilterAll) CheckFragment ¶
func (HolderFilterAll) CheckIndex ¶
func (HolderFilterAll) CheckIndex(string) (bool, bool)
type HolderFilterParams ¶
type HolderInfo ¶
type HolderInfo struct { FragmentInfo map[string]FragmentInfo FragmentNames []string }
type HolderOperator ¶
type HolderOperator interface { HolderFilter HolderProcess }
HolderOperator is both a filter and a process. This is the general form of "I want to do something to some part of a holder."
type HolderOpts ¶
type HolderOpts struct { // ReadOnly indicates that this holder's contents should not produce // disk writes under any circumstances. It must be set before Open // is called, and changing it is not supported. ReadOnly bool // If Inspect is set, we'll try to obtain additional information // about fragments when opening them. Inspect bool }
type HolderProcess ¶
type HolderProcess interface { ProcessIndex(*Index) error ProcessField(*Field) error ProcessView(*view) error ProcessFragment(*fragment) error }
HolderProcess represents something that has operations which can be performed on indexes, fields, views, and/or fragments.
type HolderProcessNone ¶
type HolderProcessNone struct{}
HolderProcessNone is a placeholder type which does nothing for the process functions. You can embed it to make a HolderOperator which does nothing, or embed it and provide your own ProcessFragment to do just that.
func (HolderProcessNone) ProcessField ¶
func (HolderProcessNone) ProcessField(*Field) error
func (HolderProcessNone) ProcessFragment ¶
func (HolderProcessNone) ProcessFragment(*fragment) error
func (HolderProcessNone) ProcessIndex ¶
func (HolderProcessNone) ProcessIndex(*Index) error
func (HolderProcessNone) ProcessView ¶
func (HolderProcessNone) ProcessView(*view) error
type ImportColumnAttrsRequest ¶
type ImportColumnAttrsRequest struct { AttrKey string ColumnIDs []uint64 AttrVals []string Shard int64 Index string IndexCreatedAt int64 }
ImportColumnAttrsRequest describes the import request structure for a ColumnAttr import
type ImportOption ¶
type ImportOption func(*ImportOptions) error
ImportOption is a functional option type for API.Import.
func OptImportOptionsClear ¶
func OptImportOptionsClear(c bool) ImportOption
OptImportOptionsClear is a functional option on ImportOption used to specify whether the import is a set or clear operation.
func OptImportOptionsIgnoreKeyCheck ¶
func OptImportOptionsIgnoreKeyCheck(b bool) ImportOption
OptImportOptionsIgnoreKeyCheck is a functional option on ImportOption used to specify whether key check should be ignored.
func OptImportOptionsPresorted ¶
func OptImportOptionsPresorted(b bool) ImportOption
type ImportOptions ¶
ImportOptions holds the options for the API.Import method.
TODO(2.0) we have entirely missed the point of functional options by exporting this structure. If it needs to be exported for some reason, we should consider not using functional options here which just adds complexity.
type ImportRequest ¶
type ImportRequest struct { Index string IndexCreatedAt int64 Field string FieldCreatedAt int64 Shard uint64 RowIDs []uint64 ColumnIDs []uint64 RowKeys []string ColumnKeys []string Timestamps []int64 }
ImportRequest describes the import request structure for an import.
func (*ImportRequest) ValidateWithTimestamp ¶
func (ir *ImportRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error
ValidateWithTimestamp ensures that the payload of the request is valid.
type ImportResponse ¶
type ImportResponse struct {
Err string
}
ImportResponse is the structured response of an import.
type ImportRoaringRequest ¶
type ImportRoaringRequest struct { IndexCreatedAt int64 FieldCreatedAt int64 Clear bool Action string // [set, clear, overwrite] Block int Views map[string][]byte }
ImportRoaringRequest describes the import request structure for an import containing roaring-encoded data.
func (*ImportRoaringRequest) ValidateWithTimestamp ¶
func (irr *ImportRoaringRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error
ValidateWithTimestamp ensures that the payload of the request is valid.
type ImportValueRequest ¶
type ImportValueRequest struct { Index string IndexCreatedAt int64 Field string FieldCreatedAt int64 // if Shard is MaxUint64 (an impossible shard value), this // indicates that the column IDs may come from multiple shards. Shard uint64 ColumnIDs []uint64 ColumnKeys []string Values []int64 FloatValues []float64 StringValues []string }
ImportValueRequest describes the import request structure for a value (BSI) import.
func (*ImportValueRequest) Len ¶
func (ivr *ImportValueRequest) Len() int
func (*ImportValueRequest) Less ¶
func (ivr *ImportValueRequest) Less(i, j int) bool
func (*ImportValueRequest) Swap ¶
func (ivr *ImportValueRequest) Swap(i, j int)
func (*ImportValueRequest) Validate ¶
func (ivr *ImportValueRequest) Validate() error
Validate ensures that the payload of the request is valid.
func (*ImportValueRequest) ValidateWithTimestamp ¶
func (ivr *ImportValueRequest) ValidateWithTimestamp(indexCreatedAt, fieldCreatedAt int64) error
ValidateWithTimestamp ensures that the payload of the request is valid.
type InMemTransactionStore ¶
type InMemTransactionStore struct {
// contains filtered or unexported fields
}
InMemTransactionStore does not persist transaction data and is only useful for testing.
func NewInMemTransactionStore ¶
func NewInMemTransactionStore() *InMemTransactionStore
func (*InMemTransactionStore) Get ¶
func (s *InMemTransactionStore) Get(id string) (*Transaction, error)
func (*InMemTransactionStore) List ¶
func (s *InMemTransactionStore) List() (map[string]*Transaction, error)
func (*InMemTransactionStore) Put ¶
func (s *InMemTransactionStore) Put(trns *Transaction) error
func (*InMemTransactionStore) Remove ¶
func (s *InMemTransactionStore) Remove(id string) (*Transaction, error)
type InMemTranslateStore ¶
type InMemTranslateStore struct {
// contains filtered or unexported fields
}
InMemTranslateStore is an in-memory storage engine for mapping keys to int values.
func NewInMemTranslateStore ¶
func NewInMemTranslateStore(index, field string, partitionID, partitionN int) *InMemTranslateStore
NewInMemTranslateStore returns a new instance of InMemTranslateStore.
func (*InMemTranslateStore) Close ¶
func (s *InMemTranslateStore) Close() error
func (*InMemTranslateStore) EntryReader ¶
func (s *InMemTranslateStore) EntryReader(ctx context.Context, offset uint64) (TranslateEntryReader, error)
EntryReader returns an error. Replication is not supported.
func (*InMemTranslateStore) ForceSet ¶
func (s *InMemTranslateStore) ForceSet(id uint64, key string) error
ForceSet writes the id/key pair to the db. Used by replication.
func (*InMemTranslateStore) MaxID ¶
func (s *InMemTranslateStore) MaxID() (uint64, error)
MaxID returns the highest identifier in the store.
func (*InMemTranslateStore) PartitionID ¶
func (s *InMemTranslateStore) PartitionID() int
PartitionID returns the partition id the store was initialized with.
func (*InMemTranslateStore) ReadFrom ¶
func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error)
ReadFrom implements io.ReaderFrom. It's not efficient or careful, but we don't expect to use InMemTranslateStore much, it's mostly there to avoid disk load during testing.
func (*InMemTranslateStore) ReadOnly ¶
func (s *InMemTranslateStore) ReadOnly() bool
ReadOnly returns true if the store is in read-only mode.
func (*InMemTranslateStore) SetReadOnly ¶
func (s *InMemTranslateStore) SetReadOnly(v bool)
SetReadOnly toggles the read-only mode of the store.
func (*InMemTranslateStore) TranslateID ¶
func (s *InMemTranslateStore) TranslateID(id uint64) (string, error)
TranslateID converts an integer ID to a string key. Returns a blank string if ID does not exist.
func (*InMemTranslateStore) TranslateIDs ¶
func (s *InMemTranslateStore) TranslateIDs(ids []uint64) ([]string, error)
TranslateIDs converts a list of integer IDs to a list of string keys.
func (*InMemTranslateStore) TranslateKey ¶
func (s *InMemTranslateStore) TranslateKey(key string, writable bool) (uint64, error)
TranslateKey converts a string key to an integer ID. If key does not have an associated id then one is created.
func (*InMemTranslateStore) TranslateKeys ¶
func (s *InMemTranslateStore) TranslateKeys(keys []string, writable bool) (_ []uint64, err error)
TranslateKeys converts a string key to an integer ID. If key does not have an associated id then one is created.
func (*InMemTranslateStore) WriteNotify ¶
func (s *InMemTranslateStore) WriteNotify() <-chan struct{}
WriteNotify returns a channel that is closed when a new entry is written.
type Index ¶
type Index struct { Stats stats.StatsClient // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc // contains filtered or unexported fields }
Index represents a container for fields.
func (*Index) AvailableShards ¶
AvailableShards returns a bitmap of all shards with data in the index.
func (*Index) ColumnAttrStore ¶
ColumnAttrStore returns the storage for column attributes.
func (*Index) CreateField ¶
func (i *Index) CreateField(name string, opts ...FieldOption) (*Field, error)
CreateField creates a field.
func (*Index) CreateFieldIfNotExists ¶
func (i *Index) CreateFieldIfNotExists(name string, opts ...FieldOption) (*Field, error)
CreateFieldIfNotExists creates a field with the given options if it doesn't exist.
func (*Index) DeleteField ¶
DeleteField removes a field from the index.
func (*Index) OpenWithTimestamp ¶
OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (*Index) Options ¶
func (i *Index) Options() IndexOptions
Options returns all options for this index.
func (*Index) TranslateStore ¶
func (i *Index) TranslateStore(partitionID int) TranslateStore
TranslateStore returns the translation store for a given partition.
func (*Index) TranslateStorePath ¶
TranslateStorePath returns the translation database path for a partition.
type IndexInfo ¶
type IndexInfo struct { Name string `json:"name"` CreatedAt int64 `json:"createdAt,omitempty"` Options IndexOptions `json:"options"` Fields []*FieldInfo `json:"fields"` ShardWidth uint64 `json:"shardWidth"` }
IndexInfo represents schema information for an index.
type IndexOptions ¶
IndexOptions represents options to set when initializing an index.
type IndexStatus ¶
type IndexStatus struct { Name string CreatedAt int64 Fields []*FieldStatus }
IndexStatus is an internal message representing the contents of an index.
type IndexTranslateOffsetMap ¶
type IndexTranslateOffsetMap struct { Partitions map[int]uint64 `json:"partitions"` Fields map[string]uint64 `json:"fields"` }
func NewIndexTranslateOffsetMap ¶
func NewIndexTranslateOffsetMap() *IndexTranslateOffsetMap
type InspectRequest ¶
type InspectRequest struct { HolderFilterParams InspectRequestParams }
InspectRequest represents a request for a possibly-partial holder inspection, using a provided holder filter and inspect-specific parameters.
type InspectRequestParams ¶
type InspectRequestParams struct { Containers bool // include container details Checksum bool // perform checksums }
InspectRequestParams represents the parts of an InspectRequest that aren't generic holder filtering attributes.
type InspectResponse ¶
type InspectResponse struct { Fragments []struct { Index string Field string View string Shard int64 Path string Info *FragmentInfo } }
InspectResponse contains the structured results for an InspectRequest. It may some day be expanded to include metadata about views or indexes.
type InternalClient ¶
type InternalClient interface { InternalQueryClient MaxShardByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) PostSchema(ctx context.Context, uri *URI, s *Schema, remote bool) error CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, shard uint64) ([]*Node, error) Nodes(ctx context.Context) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *QueryRequest) (*QueryResponse, error) Import(ctx context.Context, index, field string, shard uint64, bits []Bit, opts ...ImportOption) error ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureField(ctx context.Context, indexName string, fieldName string) error EnsureFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error ImportValue(ctx context.Context, index, field string, shard uint64, vals []FieldValue, opts ...ImportOption) error ImportValueK(ctx context.Context, index, field string, vals []FieldValue, opts ...ImportOption) error ImportValue2(ctx context.Context, req *ImportValueRequest, options *ImportOptions) error ExportCSV(ctx context.Context, index, field string, shard uint64, w io.Writer) error CreateField(ctx context.Context, index, field string) error CreateFieldWithOptions(ctx context.Context, index, field string, opt FieldOptions) error FragmentBlocks(ctx context.Context, uri *URI, index, field, view string, shard uint64) ([]FragmentBlock, error) BlockData(ctx context.Context, uri *URI, index, field, view string, shard uint64, block int) ([]uint64, []uint64, error) ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) RetrieveTranslatePartitionFromURI(ctx context.Context, index string, partition int, uri URI) (io.ReadCloser, error) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error ImportColumnAttrs(ctx context.Context, uri *URI, index string, req *ImportColumnAttrsRequest) error StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) FinishTransaction(ctx context.Context, id string) (*Transaction, error) Transactions(ctx context.Context) (map[string]*Transaction, error) GetTransaction(ctx context.Context, id string) (*Transaction, error) }
InternalClient should be implemented by any struct that enables any transport between nodes TODO: Refactor Note from Travis: Typically an interface containing more than two or three methods is an indication that something hasn't been architected correctly. While I understand that putting the entire Client behind an interface might require this many methods, I don't want to let it go unquestioned. Another note from Travis: I think we eventually want to unify `InternalClient` with the `go-pilosa` client. Doing that may obviate the need to refactor this.
type InternalQueryClient ¶
type InternalQueryClient interface { QueryNode(ctx context.Context, uri *URI, index string, queryRequest *QueryRequest) (*QueryResponse, error) // Trasnlate keys on the particular node. The parameter writable informs TranslateStore if we can generate a new ID if any of keys does not exist. TranslateKeysNode(ctx context.Context, uri *URI, index, field string, keys []string, writable bool) ([]uint64, error) TranslateIDsNode(ctx context.Context, uri *URI, index, field string, id []uint64) ([]string, error) }
InternalQueryClient is the internal interface for querying a node.
type Message ¶
type Message interface{}
Message is the interface implemented by all core pilosa types which can be serialized to messages. TODO add at least a single "isMessage()" method.
type MessageProcessingError ¶
type MessageProcessingError struct {
Err error
}
MessageProcessingError is an error indicating that a cluster message could not be processed.
func (MessageProcessingError) Cause ¶
func (err MessageProcessingError) Cause() error
Cause allows the error to be unwrapped.
func (MessageProcessingError) Error ¶
func (err MessageProcessingError) Error() string
func (MessageProcessingError) Unwrap ¶
func (err MessageProcessingError) Unwrap() error
Unwrap allows the error to be unwrapped.
type MultiTranslateEntryReader ¶
type MultiTranslateEntryReader struct {
// contains filtered or unexported fields
}
MultiTranslateEntryReader reads from multiple TranslateEntryReader instances and merges them into a single reader.
func NewMultiTranslateEntryReader ¶
func NewMultiTranslateEntryReader(ctx context.Context, readers []TranslateEntryReader) *MultiTranslateEntryReader
NewMultiTranslateEntryReader returns a new instance of MultiTranslateEntryReader.
func (*MultiTranslateEntryReader) Close ¶
func (r *MultiTranslateEntryReader) Close() error
Close stops the reader & child readers and waits for all goroutines to stop.
func (*MultiTranslateEntryReader) ReadEntry ¶
func (r *MultiTranslateEntryReader) ReadEntry(entry *TranslateEntry) error
ReadEntry reads the next available entry into entry. Returns an error if any of the child readers error. Returns io.EOF if reader is closed.
type Node ¶
type Node struct { ID string `json:"id"` URI URI `json:"uri"` GRPCURI URI `json:"grpc-uri"` IsCoordinator bool `json:"isCoordinator"` State string `json:"state"` }
Node represents a node in the cluster.
type NodeEvent ¶
type NodeEvent struct { Event NodeEventType Node *Node }
NodeEvent is a single event related to node activity in the cluster.
type NodeEventType ¶
type NodeEventType int
NodeEventType are the types of node events.
const ( NodeJoin NodeEventType = iota NodeLeave NodeUpdate )
Constant node event types.
type NodeStateMessage ¶
type NodeStateMessage struct { NodeID string `protobuf:"bytes,1,opt,name=NodeID,proto3" json:"NodeID,omitempty"` State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"` }
NodeStateMessage is an internal message for broadcasting a node's state.
type NodeStatus ¶
type NodeStatus struct { Node *Node Indexes []*IndexStatus Schema *Schema }
NodeStatus is an internal message representing the contents of a node.
type Nodes ¶
type Nodes []*Node
Nodes represents a list of nodes.
func (Nodes) ContainsID ¶
ContainsID returns true if host matches one of the node's id.
type NotFoundError ¶
type NotFoundError struct {
// contains filtered or unexported fields
}
NotFoundError wraps an error value to signify that a resource was not found such that in an HTTP scenario, http.StatusNotFound would be returned.
type OpenTransactionStoreFunc ¶
type OpenTransactionStoreFunc func(path string) (TransactionStore, error)
type OpenTranslateReaderFunc ¶
type OpenTranslateReaderFunc func(ctx context.Context, nodeURL string, offsets TranslateOffsetMap) (TranslateEntryReader, error)
OpenTranslateReaderFunc represents a function for instantiating and opening a TranslateStore.
type OpenTranslateStoreFunc ¶
type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error)
OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore.
type PairField ¶
PairField is a Pair with its associated field.
func (PairField) MarshalJSON ¶
MarshalJSON marshals PairField into a JSON-encoded byte slice, excluding `Field`.
type Pairs ¶
type Pairs []Pair
Pairs is a sortable slice of Pair objects.
func (*Pairs) Pop ¶
func (p *Pairs) Pop() interface{}
Pop removes the minimum element from the Pair slice.
type PairsField ¶
PairsField is a Pairs object with its associated field.
func (PairsField) MarshalJSON ¶
func (p PairsField) MarshalJSON() ([]byte, error)
MarshalJSON marshals PairsField into a JSON-encoded byte slice, excluding `Field`.
func (*PairsField) ToRows ¶
func (p *PairsField) ToRows(callback func(*pb.RowResponse) error) error
ToRows implements the ToRowser interface.
func (*PairsField) ToTable ¶
func (p *PairsField) ToTable() (*pb.TableResponse, error)
ToTable implements the ToTabler interface.
type PreconditionFailedError ¶
type PreconditionFailedError struct {
// contains filtered or unexported fields
}
type QueryRequest ¶
type QueryRequest struct { // Index to execute query against. Index string // The query string to parse and execute. Query string // The shards to include in the query execution. // If empty, all shards are included. Shards []uint64 // Return column attributes, if true. ColumnAttrs bool // Do not return row attributes, if true. ExcludeRowAttrs bool // Do not return columns, if true. ExcludeColumns bool // If true, indicates that query is part of a larger distributed query. // If false, this request is on the originating node. Remote bool // Should we profile this query? Profile bool // Additional data associated with the query, in cases where there's // row-style inputs for precomputed values. EmbeddedData []*Row }
QueryRequest represent a request to process a query.
type QueryResponse ¶
type QueryResponse struct { // Result for each top-level query call. // The result type differs depending on the query; types // include: Row, RowIdentifiers, GroupCounts, SignedRow, // ValCount, Pair, Pairs, bool, uint64. Results []interface{} // Set of column attribute objects matching IDs returned in Result. ColumnAttrSets []*ColumnAttrSet // Error during parsing or execution. Err error // Profiling data, if any Profile *tracing.Profile }
QueryResponse represent a response from a processed query.
func (*QueryResponse) MarshalJSON ¶
func (resp *QueryResponse) MarshalJSON() ([]byte, error)
MarshalJSON marshals QueryResponse into a JSON-encoded byte slice
type RecalculateCaches ¶
type RecalculateCaches struct{}
RecalculateCaches is an internal message for recalculating all caches within a holder.
type ResizeInstruction ¶
type ResizeInstruction struct { JobID int64 Node *Node Coordinator *Node Sources []*ResizeSource TranslationSources []*TranslationResizeSource NodeStatus *NodeStatus ClusterStatus *ClusterStatus }
ResizeInstruction contains the instruction provided to a node during a cluster resize operation.
type ResizeInstructionComplete ¶
ResizeInstructionComplete is an internal message to the coordinator indicating that the resize instructions performed on a single node have completed.
type ResizeSource ¶
type ResizeSource struct { Node *Node `protobuf:"bytes,1,opt,name=Node" json:"Node,omitempty"` Index string `protobuf:"bytes,2,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,3,opt,name=Field,proto3" json:"Field,omitempty"` View string `protobuf:"bytes,4,opt,name=View,proto3" json:"View,omitempty"` Shard uint64 `protobuf:"varint,5,opt,name=Shard,proto3" json:"Shard,omitempty"` }
ResizeSource is the source of data for a node acting on a ResizeInstruction.
type Row ¶
type Row struct { // String keys translated to/from segment columns. Keys []string // Attributes associated with the row. Attrs map[string]interface{} // contains filtered or unexported fields }
Row is a set of integers (the associated columns), and attributes which are arbitrary key/value pairs storing metadata about what the row represents.
func NewRowFromBitmap ¶
NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This transposes; data that was in any shard for Row 0 is now considered shard 0, etcetera.
func NewRowFromRoaring ¶
NewRowFromRoaring parses a roaring data file as a row, dividing it into bitmaps and rowSegments based on shard width.
func (*Row) Difference ¶
Difference returns the diff of r and other.
func (*Row) GenericBinaryOp ¶
func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row
GenericBinaryOp returns the output of a generic op on r and other.
func (*Row) GenericCount ¶
func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64
GenericCount applies an op to lots of things.
func (*Row) GenericNaryOp ¶
func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row
GenericNaryOp returns the output of an nary op on r and others.
func (*Row) GenericUnaryOp ¶
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row
GenericUnaryOp returns the results of a generic op on r.
func (*Row) MarshalJSON ¶
MarshalJSON returns a JSON-encoded byte slice of r.
func (*Row) Segments ¶
func (r *Row) Segments() []rowSegment
Segments returns a list of all segments in the row.
func (*Row) Shift ¶
Shift returns the bitwise shift of r by n bits. Currently only positive shift values are supported.
NOTE: the Shift method is currently unsupported, and is considerred to be incorrect. Please DO NOT use it. We are leaving it here in case someone internally wants to use it with the understanding that the results may be incorrect.
Why unsupported? For a full description, see: https://github.com/molecula/pilosa/issues/403. In short, the current implementation will shift a bit at the edge of a shard out of the shard and into a container which is assumed to be an invalid container for the shard. So for example, shifting the last bit of shard 0 (containers 0-15) will shift that bit out to container 16. While this "sort of" works, it breaks an assumption about containers, and might stop working in the future if that assumption is enforced.
func (*Row) ToRows ¶
func (r *Row) ToRows(callback func(*pb.RowResponse) error) error
ToRows implements the ToRowser interface.
func (*Row) ToTable ¶
func (r *Row) ToTable() (*pb.TableResponse, error)
ToTable implements the ToTabler interface.
type RowIDs ¶
type RowIDs []uint64
RowIDs is a query return type for just uint64 row ids. It should only be used internally (since RowIdentifiers is the external return type), but it is exported because the proto package needs access to it.
type RowIdentifiers ¶
type RowIdentifiers struct { Rows []uint64 `json:"rows"` Keys []string `json:"keys,omitempty"` // contains filtered or unexported fields }
RowIdentifiers is a return type for a list of row ids or row keys. The names `Rows` and `Keys` are meant to follow the same convention as the Row query which returns `Columns` and `Keys`. TODO: Rename this to something better. Anything.
func (*RowIdentifiers) Field ¶
func (r *RowIdentifiers) Field() string
Field returns the field name associated to the row.
func (RowIdentifiers) ToRows ¶
func (r RowIdentifiers) ToRows(callback func(*pb.RowResponse) error) error
ToRows implements the ToRowser interface.
func (RowIdentifiers) ToTable ¶
func (r RowIdentifiers) ToTable() (*pb.TableResponse, error)
ToTable implements the ToTabler interface.
type Schema ¶
type Schema struct {
Indexes []*IndexInfo `json:"indexes"`
}
Schema contains information about indexes and their configuration.
type Serializer ¶
Serializer is an interface for serializing pilosa types to bytes and back.
type Server ¶
type Server struct {
// contains filtered or unexported fields
}
Server represents a holder wrapped by a running HTTP server.
func NewServer ¶
func NewServer(opts ...ServerOption) (*Server, error)
NewServer returns a new instance of Server.
func (*Server) FinishTransaction ¶
func (*Server) GetTransaction ¶
func (*Server) InternalClient ¶
func (s *Server) InternalClient() InternalClient
func (*Server) StartTransaction ¶
func (*Server) SyncData ¶
SyncData manually invokes the anti entropy process which makes sure that this node has the data from all replicas across the cluster.
func (*Server) Transactions ¶
type ServerOption ¶
ServerOption is a functional option type for pilosa.Server
func OptServerAntiEntropyInterval ¶
func OptServerAntiEntropyInterval(interval time.Duration) ServerOption
OptServerAntiEntropyInterval is a functional option on Server used to set the anti-entropy interval.
func OptServerAttrStoreFunc ¶
func OptServerAttrStoreFunc(af func(string) AttrStore) ServerOption
OptServerAttrStoreFunc is a functional option on Server used to provide the function to use to generate a new attribute store.
func OptServerClusterDisabled ¶
func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption
OptServerClusterDisabled tells the server whether to use a static cluster with the defined hosts. Mostly used for testing.
func OptServerClusterHasher ¶
func OptServerClusterHasher(h Hasher) ServerOption
OptServerClusterHasher is a functional option on Server used to specify the consistent hash algorithm for data location within the cluster.
func OptServerDataDir ¶
func OptServerDataDir(dir string) ServerOption
OptServerDataDir is a functional option on Server used to set the data directory.
func OptServerDiagnosticsInterval ¶
func OptServerDiagnosticsInterval(dur time.Duration) ServerOption
OptServerDiagnosticsInterval is a functional option on Server used to specify the duration between diagnostic checks.
func OptServerExecutorPoolSize ¶
func OptServerExecutorPoolSize(size int) ServerOption
func OptServerGCNotifier ¶
func OptServerGCNotifier(gcn GCNotifier) ServerOption
OptServerGCNotifier is a functional option on Server used to set the garbage collection notification source.
func OptServerGRPCURI ¶
func OptServerGRPCURI(uri *URI) ServerOption
OptServerGRPCURI is a functional option on Server used to set the server gRPC URI.
func OptServerInternalClient ¶
func OptServerInternalClient(c InternalClient) ServerOption
OptServerInternalClient is a functional option on Server used to set the implementation of InternalClient.
func OptServerIsCoordinator ¶
func OptServerIsCoordinator(is bool) ServerOption
OptServerIsCoordinator is a functional option on Server used to specify whether or not this server is the coordinator.
func OptServerLogger ¶
func OptServerLogger(l logger.Logger) ServerOption
OptServerLogger is a functional option on Server used to set the logger.
func OptServerLongQueryTime ¶
func OptServerLongQueryTime(dur time.Duration) ServerOption
OptServerLongQueryTime is a functional option on Server used to set long query duration.
func OptServerMaxWritesPerRequest ¶
func OptServerMaxWritesPerRequest(n int) ServerOption
OptServerMaxWritesPerRequest is a functional option on Server used to set the maximum number of writes allowed per request.
func OptServerMetricInterval ¶
func OptServerMetricInterval(dur time.Duration) ServerOption
OptServerMetricInterval is a functional option on Server used to set the interval between metric samples.
func OptServerNodeDownRetries ¶
func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption
OptServerNodeDownRetries is a functional option on Server used to specify the retries and sleep duration for node down checks.
func OptServerNodeID ¶
func OptServerNodeID(nodeID string) ServerOption
OptServerNodeID is a functional option on Server used to set the server node ID.
func OptServerOpenTranslateReader ¶
func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption
OptServerOpenTranslateReader is a functional option on Server used to specify the remote translation data reader.
func OptServerOpenTranslateStore ¶
func OptServerOpenTranslateStore(fn OpenTranslateStoreFunc) ServerOption
OptServerOpenTranslateStore is a functional option on Server used to specify the translation data store type.
func OptServerPrimaryTranslateStore ¶
func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption
OptServerPrimaryTranslateStore has been deprecated.
func OptServerReplicaN ¶
func OptServerReplicaN(n int) ServerOption
OptServerReplicaN is a functional option on Server used to set the number of replicas.
func OptServerSerializer ¶
func OptServerSerializer(ser Serializer) ServerOption
OptServerSerializer is a functional option on Server used to set the serializer.
func OptServerStatsClient ¶
func OptServerStatsClient(sc stats.StatsClient) ServerOption
OptServerStatsClient is a functional option on Server used to specify the stats client.
func OptServerSystemInfo ¶
func OptServerSystemInfo(si SystemInfo) ServerOption
OptServerSystemInfo is a functional option on Server used to set the system information source.
func OptServerURI ¶
func OptServerURI(uri *URI) ServerOption
OptServerURI is a functional option on Server used to set the server URI.
type SetCoordinatorMessage ¶
type SetCoordinatorMessage struct {
New *Node
}
SetCoordinatorMessage is an internal message instructing nodes to honor a new coordinator.
type SignedRow ¶
type SignedRow struct { Neg *Row `json:"neg"` Pos *Row `json:"pos"` // contains filtered or unexported fields }
SignedRow represents a signed *Row with two (neg/pos) *Rows.
type SnapshotQueue ¶
type SnapshotQueue interface { Immediate(*fragment) error Enqueue(*fragment) Await(*fragment) error ScanHolder(*Holder, chan struct{}) Stop() }
snapshotQueue is a thing which can handle enqueuing snapshots. A snapshot queue distinguishes between high-priority requests, which get satisfied by the next available worker, and regular requests, which get enqueued if there's space in the queue, and otherwise dropped. There's also a separate background task to scan a holder for fragments which may need snapshots, but which is processed only when the queue is empty, and only slowly. "Await" awaits an existing snapshot if one is already enqueued. "Immediate" tries to do one right away. (If one's already enqueued, this can leave it in the queue, which will ignore anything that shows up with the request flag cleared.)
Await, Enqueue, and Immediate should be called only with the fragment lock held.
If you create a queue, it should get stopped at some point. The atomicSnapshotQueue implementation used as defaultSnapshotQueue has a Start function which will tell you whether it actually started a queue. This logic exists because in a normal server case, you probably want the queue to be shut down as part of server shutdown, but if you're running cluster tests, you probably want to start and shop the queue as part of the test, not stop it when any server terminates.
It's less likely to be desireable to start/stop individual queues, because fragments use the defaultSnapshotQueue anyway. This design needs revisiting.
type SystemInfo ¶
type SystemInfo interface { Uptime() (uint64, error) Platform() (string, error) Family() (string, error) OSVersion() (string, error) KernelVersion() (string, error) MemFree() (uint64, error) MemTotal() (uint64, error) MemUsed() (uint64, error) CPUModel() string CPUCores() (physical int, logical int, err error) CPUMHz() (int, error) CPUArch() string }
SystemInfo collects information about the host OS.
type TimeQuantum ¶
type TimeQuantum string
TimeQuantum represents a time granularity for time-based bitmaps.
func (TimeQuantum) HasDay ¶
func (q TimeQuantum) HasDay() bool
HasDay returns true if the quantum contains a 'D' unit.
func (TimeQuantum) HasHour ¶
func (q TimeQuantum) HasHour() bool
HasHour returns true if the quantum contains a 'H' unit.
func (TimeQuantum) HasMonth ¶
func (q TimeQuantum) HasMonth() bool
HasMonth returns true if the quantum contains a 'M' unit.
func (TimeQuantum) HasYear ¶
func (q TimeQuantum) HasYear() bool
HasYear returns true if the quantum contains a 'Y' unit.
func (*TimeQuantum) Set ¶
func (q *TimeQuantum) Set(value string) error
Set sets the time quantum value.
func (TimeQuantum) String ¶
func (q TimeQuantum) String() string
func (TimeQuantum) Type ¶
func (q TimeQuantum) Type() string
Type returns the type of a time quantum value.
func (TimeQuantum) Valid ¶
func (q TimeQuantum) Valid() bool
Valid returns true if q is a valid time quantum value.
type Topology ¶
type Topology struct {
// contains filtered or unexported fields
}
Topology represents the list of hosts in the cluster.
func (*Topology) ContainsID ¶
ContainsID returns true if id matches one of the topology's IDs.
type Transaction ¶
type Transaction struct { // ID is an arbitrary string identifier. All transactions must have a unique ID. ID string `json:"id"` // Active notes whether an exclusive transaction is active, or // still pending (if other active transactions exist). All // non-exclusive transactions are always active. Active bool `json:"active"` // Exclusive is set to true for transactions which can only become active when no other // transactions exist. Exclusive bool `json:"exclusive"` // Timeout is the minimum idle time for which this transaction should continue to exist. Timeout time.Duration `json:"timeout"` // Deadline is calculated from Timeout. TODO reset deadline each time there is activity // on the transaction. (we can't do this until there is some method of associating a // request/call with a transaction) Deadline time.Time `json:"deadline"` // Stats track statistics for the transaction. Not yet used. Stats TransactionStats `json:"stats"` }
Transaction contains information related to a block of work that needs to be tracked and spans multiple API calls.
func (*Transaction) Copy ¶
func (trns *Transaction) Copy() *Transaction
func (*Transaction) MarshalJSON ¶
func (trns *Transaction) MarshalJSON() ([]byte, error)
func (*Transaction) UnmarshalJSON ¶
func (trns *Transaction) UnmarshalJSON(b []byte) error
type TransactionManager ¶
TransactionManager enforces the rules for transactions on a single node. It is goroutine-safe. It should be created by a call to NewTransactionManager where it takes a TransactionStore. If logging is desired, Log should be set before an instance of TransactionManager is used.
func NewTransactionManager ¶
func NewTransactionManager(store TransactionStore) *TransactionManager
NewTransactionManager creates a new TransactionManager with the given store, and starts a deadline-checker in a goroutine.
func (*TransactionManager) Finish ¶
func (tm *TransactionManager) Finish(ctx context.Context, id string) (*Transaction, error)
Finish completes and removes a transaction, returning the completed transaction (so that the caller can e.g. view the Stats)
func (*TransactionManager) Get ¶
func (tm *TransactionManager) Get(ctx context.Context, id string) (*Transaction, error)
Get retrieves the transaction with the given ID. Returns ErrTransactionNotFound if there isn't one.
func (*TransactionManager) List ¶
func (tm *TransactionManager) List(ctx context.Context) (map[string]*Transaction, error)
List returns map of all transactions by their ID. It is a copy and so may be retained and modified by the caller.
func (*TransactionManager) ResetDeadline ¶
func (tm *TransactionManager) ResetDeadline(ctx context.Context, id string) (*Transaction, error)
ResetDeadline updates the deadline for the transaction with the given ID to be equal to the current time plus the transaction's timeout.
func (*TransactionManager) Start ¶
func (tm *TransactionManager) Start(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error)
Start starts a new transaction with the given parameters. If an exclusive transaction is pending or in progress, ErrTransactionExclusive is returned. If a transaction with the same id already exists, that transaction is returned along with ErrTransactionExists. If there is no error, the created transaction is returned—this is primarily so that the caller can discover if an exclusive transaction has been made immediately active or if they need to poll.
type TransactionMessage ¶
type TransactionMessage struct { Transaction *Transaction Action string }
type TransactionStats ¶
type TransactionStats struct{}
type TransactionStore ¶
type TransactionStore interface { // Put stores a new transaction or replaces an existing transaction with the given one. Put(trns *Transaction) error // Get retrieves the transaction at id or returns ErrTransactionNotFound if there isn't one. Get(id string) (*Transaction, error) // List returns a map of all transactions by ID. The map must be safe to modify by the caller. List() (map[string]*Transaction, error) // Remove deletes the transaction from the store. It must return ErrTransactionNotFound if there isn't one. Remove(id string) (*Transaction, error) }
TransactionStore declares the functionality which a store for Pilosa transactions must implement.
func OpenInMemTransactionStore ¶
func OpenInMemTransactionStore(path string) (TransactionStore, error)
type TranslateEntry ¶
type TranslateEntry struct { Index string `json:"index,omitempty"` Field string `json:"field,omitempty"` ID uint64 `json:"id,omitempty"` Key string `json:"key,omitempty"` }
TranslateEntry represents a key/ID pair from a TranslateStore.
type TranslateEntryReader ¶
type TranslateEntryReader interface { io.Closer ReadEntry(entry *TranslateEntry) error }
TranslateEntryReader represents a stream of translation entries.
type TranslateIDsRequest ¶
TranslateIDsRequest describes the structure of a request for a batch of id translations.
type TranslateIDsResponse ¶
type TranslateIDsResponse struct {
Keys []string
}
TranslateIDsResponse is the structured response of a id translation request.
type TranslateKeysRequest ¶
type TranslateKeysRequest struct { Index string Field string Keys []string // it's a awkward name, just to keep backward compatibility with go-pilosa and idk. NotWritable bool }
TranslateKeysRequest describes the structure of a request for a batch of key translations.
type TranslateKeysResponse ¶
type TranslateKeysResponse struct {
IDs []uint64
}
TranslateKeysResponse is the structured response of a key translation request.
type TranslateOffsetMap ¶
type TranslateOffsetMap map[string]*IndexTranslateOffsetMap
TranslateOffsetMap maintains a set of offsets for both indexes & fields.
func (TranslateOffsetMap) FieldOffset ¶
func (m TranslateOffsetMap) FieldOffset(index, name string) uint64
FieldOffset returns the offset for the given field.
func (TranslateOffsetMap) IndexPartitionOffset ¶
func (m TranslateOffsetMap) IndexPartitionOffset(name string, partitionID int) uint64
IndexOffset returns the offset for the given index.
func (TranslateOffsetMap) SetFieldOffset ¶
func (m TranslateOffsetMap) SetFieldOffset(index, name string, offset uint64)
SetFieldOffset sets the offset for the given field.
func (TranslateOffsetMap) SetIndexPartitionOffset ¶
func (m TranslateOffsetMap) SetIndexPartitionOffset(name string, partitionID int, offset uint64)
SetIndexOffset sets the offset for the given index.
type TranslateStore ¶
type TranslateStore interface { io.Closer // Returns the maximum ID set on the store. MaxID() (uint64, error) // Retrieves the partition ID associated with the store. // Only applies to index stores. PartitionID() int // Sets & retrieves whether the store is read-only. ReadOnly() bool SetReadOnly(v bool) // Converts a string key to its autoincrementing integer ID value. // // Translated id must be associated with a shard in the store's partition // unless partition is set to -1. TranslateKey(key string, writable bool) (uint64, error) TranslateKeys(key []string, writable bool) ([]uint64, error) // Converts an integer ID to its associated string key. TranslateID(id uint64) (string, error) TranslateIDs(id []uint64) ([]string, error) // Forces the write of a key/id pair, even if read only. Used by replication. ForceSet(id uint64, key string) error // Returns a reader from the given ID offset. EntryReader(ctx context.Context, offset uint64) (TranslateEntryReader, error) // WriteTo ensures that the TranslateStore implements io.WriterTo. // It should write the contents of the store to the writer. WriteTo(io.Writer) (int64, error) // ReadFrom ensures that the TranslateStore implements io.ReaderFrom. // It should read from the reader and replace the data store with // the read payload. ReadFrom(io.Reader) (int64, error) }
TranslateStore is the storage for translation string-to-uint64 values. For BoltDB implementation an empty string will be converted into the sentinel byte slice:
var emptyKey = []byte{ 0x00, 0x00, 0x00, 0x4d, 0x54, 0x4d, 0x54, // MTMT 0x00, 0xc2, 0xa0, // NO-BREAK SPACE 0x00, }
func OpenInMemTranslateStore ¶
func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error)
OpenInMemTranslateStore returns a new instance of InMemTranslateStore. Implements OpenTranslateStoreFunc.
type TranslationResizeSource ¶
TranslationResizeSource is the source of translation data for a node acting on a ResizeInstruction.
type URI ¶
type URI struct { Scheme string `json:"scheme"` Host string `json:"host"` Port uint16 `json:"port"` }
URI represents a Pilosa URI. A Pilosa URI consists of three parts: 1) Scheme: Protocol of the URI. Default: http. 2) Host: Hostname or IP URI. Default: localhost. IPv6 addresses should be written in brackets, e.g., `[fd42:4201:f86b:7e09:216:3eff:fefa:ed80]`. 3) Port: Port of the URI. Default: 10101.
All parts of the URI are optional. The following are equivalent:
http://localhost:10101 http://localhost http://:10101 localhost:10101 localhost :10101
func AddressWithDefaults ¶
AddressWithDefaults converts addr into a valid address, using defaults when necessary.
func NewURIFromAddress ¶
NewURIFromAddress parses the passed address and returns a URI.
func NewURIFromHostPort ¶
NewURIFromHostPort returns a URI with specified host and port.
func (*URI) MarshalJSON ¶
MarshalJSON marshals URI into a JSON-encoded byte slice.
func (*URI) UnmarshalJSON ¶
UnmarshalJSON unmarshals a byte slice to a URI.
type URIs ¶
type URIs []URI
URIs is a convenience type representing a slice of URI.
func (URIs) HostPortStrings ¶
HostPortStrings returns a slice of host:port strings based on the slice of URI.
type UpdateCoordinatorMessage ¶
type UpdateCoordinatorMessage struct {
New *Node
}
UpdateCoordinatorMessage is an internal message for reassigning the coordinator.
type ValCount ¶
type ValCount struct { Val int64 `json:"value"` FloatVal float64 `json:"floatValue"` DecimalVal *pql.Decimal `json:"decimalValue"` Count int64 `json:"count"` }
ValCount represents a grouping of sum & count for Sum() and Average() calls. Also Min, Max....
Source Files ¶
- api.go
- apimethod_string.go
- attr.go
- broadcast.go
- cache.go
- client.go
- cluster.go
- cmd.go
- diagnostics.go
- doc.go
- event.go
- executor.go
- extension.go
- field.go
- fragment.go
- gc.go
- generation.go
- generation_nodebug.go
- handler.go
- holder.go
- index.go
- iterator.go
- metrics.go
- pilosa.go
- row.go
- server.go
- snapshotqueue.go
- time.go
- tracker.go
- transaction.go
- translate.go
- uri.go
- version.go
- view.go
Directories ¶
Path | Synopsis |
---|---|
api
|
|
Package cmd contains all the pilosa subcommand definitions (1 per file).
|
Package cmd contains all the pilosa subcommand definitions (1 per file). |
pilosa
This is the entrypoint for the Pilosa binary.
|
This is the entrypoint for the Pilosa binary. |
package ctl contains all pilosa subcommands other than 'server'.
|
package ctl contains all pilosa subcommands other than 'server'. |
encoding
|
|
Package lru implements an LRU cache.
|
Package lru implements an LRU cache. |
Package pql defines the Pilosa Query Language.
|
Package pql defines the Pilosa Query Language. |
Package roaring implements roaring bitmaps with support for incremental changes.
|
Package roaring implements roaring bitmaps with support for incremental changes. |
Package syswrap wraps syscalls (just mmap right now) in order to impose a global in-process limit on the maximum number of active mmaps.
|
Package syswrap wraps syscalls (just mmap right now) in order to impose a global in-process limit on the maximum number of active mmaps. |