cluster

package
v0.1.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 22, 2015 License: MIT, MIT Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// DefaultWriteTimeout is the default timeout for a complete write to succeed.
	DefaultWriteTimeout = 5 * time.Second

	// DefaultShardWriterTimeout is the default timeout set on shard writers.
	DefaultShardWriterTimeout = 5 * time.Second

	// DefaultShardMapperTimeout is the default timeout set on shard mappers.
	DefaultShardMapperTimeout = 5 * time.Second
)
View Source 
const MaxMessageSize = 1024 * 1024 * 1024 // 1GB

MaxMessageSize defines how large a message can be before we reject it

View Source 
const MuxHeader = 2

MuxHeader is the header byte used in the TCP mux.

Variables

View Source 
var (
	// ErrTimeout is returned when a write times out.
	ErrTimeout = errors.New("timeout")

	// ErrPartialWrite is returned when a write partially succeeds but does
	// not meet the requested consistency level.
	ErrPartialWrite = errors.New("partial write")

	// ErrWriteFailed is returned when no writes succeeded.
	ErrWriteFailed = errors.New("write failed")

	// ErrInvalidConsistencyLevel is returned when parsing the string version
	// of a consistency level.
	ErrInvalidConsistencyLevel = errors.New("invalid consistency level")
)

Functions

func ReadTLV 

func ReadTLV(r io.Reader) (byte, []byte, error)

ReadTLV reads a type-length-value record from r.

func WriteTLV 

func WriteTLV(w io.Writer, typ byte, buf []byte) error

WriteTLV writes a type-length-value record to w.

Types

type Config 

type Config struct {
	ForceRemoteShardMapping bool          `toml:"force-remote-mapping"`
	WriteTimeout            toml.Duration `toml:"write-timeout"`
	ShardWriterTimeout      toml.Duration `toml:"shard-writer-timeout"`
	ShardMapperTimeout      toml.Duration `toml:"shard-mapper-timeout"`
}

Config represents the configuration for the clustering service.

func NewConfig 

func NewConfig() Config

NewConfig returns an instance of Config with defaults.

type ConsistencyLevel 

type ConsistencyLevel int

ConsistencyLevel represent a required replication criteria before a write can be returned as successful 

const (
	// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
	ConsistencyLevelAny ConsistencyLevel = iota

	// ConsistencyLevelOne requires at least one data node acknowledged a write
	ConsistencyLevelOne

	// ConsistencyLevelOne requires a quorum of data nodes to acknowledge a write
	ConsistencyLevelQuorum

	// ConsistencyLevelAll requires all data nodes to acknowledge a write
	ConsistencyLevelAll
)

func ParseConsistencyLevel 

func ParseConsistencyLevel(level string) (ConsistencyLevel, error)

type MapShardRequest 

type MapShardRequest struct {
	// contains filtered or unexported fields
}

MapShardRequest represents the request to map a remote shard for a query.

func (*MapShardRequest) ChunkSize 

func (m *MapShardRequest) ChunkSize() int32

func (*MapShardRequest) MarshalBinary 

func (m *MapShardRequest) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*MapShardRequest) Query 

func (m *MapShardRequest) Query() string

func (*MapShardRequest) SetChunkSize 

func (m *MapShardRequest) SetChunkSize(chunkSize int32)

func (*MapShardRequest) SetQuery 

func (m *MapShardRequest) SetQuery(query string)

func (*MapShardRequest) SetShardID 

func (m *MapShardRequest) SetShardID(id uint64)

func (*MapShardRequest) ShardID 

func (m *MapShardRequest) ShardID() uint64

func (*MapShardRequest) UnmarshalBinary 

func (m *MapShardRequest) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates MapShardRequest from a binary format.

type MapShardResponse 

type MapShardResponse struct {
	// contains filtered or unexported fields
}

MapShardResponse represents the response returned from a remote MapShardRequest call

func NewMapShardResponse 

func NewMapShardResponse(code int, message string) *MapShardResponse

func (*MapShardResponse) Code 

func (r *MapShardResponse) Code() int

func (*MapShardResponse) Data 

func (r *MapShardResponse) Data() []byte

func (*MapShardResponse) Fields 

func (r *MapShardResponse) Fields() []string

func (*MapShardResponse) MarshalBinary 

func (r *MapShardResponse) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*MapShardResponse) Message 

func (r *MapShardResponse) Message() string

func (*MapShardResponse) SetCode 

func (r *MapShardResponse) SetCode(code int)

func (*MapShardResponse) SetData 

func (r *MapShardResponse) SetData(data []byte)

func (*MapShardResponse) SetFields 

func (r *MapShardResponse) SetFields(fields []string)

func (*MapShardResponse) SetMessage 

func (r *MapShardResponse) SetMessage(message string)

func (*MapShardResponse) SetTagSets 

func (r *MapShardResponse) SetTagSets(tagsets []string)

func (*MapShardResponse) TagSets 

func (r *MapShardResponse) TagSets() []string

func (*MapShardResponse) UnmarshalBinary 

func (r *MapShardResponse) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates WritePointRequest from a binary format.

type PointsWriter 

type PointsWriter struct {
	WriteTimeout time.Duration
	Logger       *log.Logger

	MetaStore interface {
		NodeID() uint64
		Database(name string) (di *meta.DatabaseInfo, err error)
		RetentionPolicy(database, policy string) (*meta.RetentionPolicyInfo, error)
		CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
	}

	TSDBStore interface {
		CreateShard(database, retentionPolicy string, shardID uint64) error
		WriteToShard(shardID uint64, points []tsdb.Point) error
	}

	ShardWriter interface {
		WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
	}

	HintedHandoff interface {
		WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
	}
	// contains filtered or unexported fields
}

PointsWriter handles writes across multiple local and remote data nodes.

func NewPointsWriter 

func NewPointsWriter() *PointsWriter

NewPointsWriter returns a new instance of PointsWriter for a node.

func (*PointsWriter) Close 

func (w *PointsWriter) Close() error

func (*PointsWriter) MapShards 

func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error)

MapShards maps the points contained in wp to a ShardMapping. If a point maps to a shard group or shard that does not currently exist, it will be created before returning the mapping.

func (*PointsWriter) Open 

func (w *PointsWriter) Open() error

func (*PointsWriter) WritePoints 

func (w *PointsWriter) WritePoints(p *WritePointsRequest) error

WritePoints writes across multiple local and remote data nodes according the consistency level.

type RemoteMapper 

type RemoteMapper struct {
	// contains filtered or unexported fields
}

RemoteMapper implements the tsdb.Mapper interface. It connects to a remote node, sends a query, and interprets the stream of data that comes back.

func NewRemoteMapper 

func NewRemoteMapper(c remoteShardConn, shardID uint64, stmt string, chunkSize int) *RemoteMapper

NewRemoteMapper returns a new remote mapper using the given connection.

func (*RemoteMapper) Close 

func (r *RemoteMapper) Close()

Close the Mapper

func (*RemoteMapper) Fields 

func (r *RemoteMapper) Fields() []string

func (*RemoteMapper) NextChunk 

func (r *RemoteMapper) NextChunk() (chunk interface{}, err error)

NextChunk returns the next chunk read from the remote node to the client.

func (*RemoteMapper) Open 

func (r *RemoteMapper) Open() (err error)

Open connects to the remote node and starts receiving data.

func (*RemoteMapper) TagSets 

func (r *RemoteMapper) TagSets() []string

type Service 

type Service struct {
	Listener net.Listener

	MetaStore interface {
		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
	}

	TSDBStore interface {
		CreateShard(database, policy string, shardID uint64) error
		WriteToShard(shardID uint64, points []tsdb.Point) error
		CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
	}

	Logger *log.Logger
	// contains filtered or unexported fields
}

Service processes data received over raw TCP connections.

func NewService 

func NewService(c Config) *Service

NewService returns a new instance of Service.

func (*Service) Close 

func (s *Service) Close() error

Close shuts down the listener and waits for all connections to finish.

func (*Service) Open 

func (s *Service) Open() error

Open opens the network listener and begins serving requests.

func (*Service) SetLogger 

func (s *Service) SetLogger(l *log.Logger)

SetLogger sets the internal logger to the logger passed in.

type ShardMapper 

type ShardMapper struct {
	ForceRemoteMapping bool // All shards treated as remote. Useful for testing.

	MetaStore interface {
		NodeID() uint64
		Node(id uint64) (ni *meta.NodeInfo, err error)
	}

	TSDBStore interface {
		CreateMapper(shardID uint64, query string, chunkSize int) (tsdb.Mapper, error)
	}
	// contains filtered or unexported fields
}

ShardMapper is responsible for providing mappers for requested shards. It is responsible for creating those mappers from the local store, or reaching out to another node on the cluster.

func NewShardMapper 

func NewShardMapper(timeout time.Duration) *ShardMapper

NewShardMapper returns a mapper of local and remote shards.

func (*ShardMapper) CreateMapper 

func (s *ShardMapper) CreateMapper(sh meta.ShardInfo, stmt string, chunkSize int) (tsdb.Mapper, error)

CreateMapper returns a Mapper for the given shard ID.

type ShardMapping 

type ShardMapping struct {
	Points map[uint64][]tsdb.Point    // The points associated with a shard ID
	Shards map[uint64]*meta.ShardInfo // The shards that have been mapped, keyed by shard ID
}

ShardMapping contains a mapping of a shards to a points.

func NewShardMapping 

func NewShardMapping() *ShardMapping

NewShardMapping creates an empty ShardMapping

func (*ShardMapping) MapPoint 

func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p tsdb.Point)

MapPoint maps a point to shard

type ShardWriter 

type ShardWriter struct {
	MetaStore interface {
		Node(id uint64) (ni *meta.NodeInfo, err error)
	}
	// contains filtered or unexported fields
}

ShardWriter writes a set of points to a shard.

func NewShardWriter 

func NewShardWriter(timeout time.Duration) *ShardWriter

NewShardWriter returns a new instance of ShardWriter.

func (*ShardWriter) Close 

func (w *ShardWriter) Close() error

func (*ShardWriter) WriteShard 

func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error

type WritePointsRequest 

type WritePointsRequest struct {
	Database         string
	RetentionPolicy  string
	ConsistencyLevel ConsistencyLevel
	Points           []tsdb.Point
}

WritePointsRequest represents a request to write point data to the cluster

func (*WritePointsRequest) AddPoint 

func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)

AddPoint adds a point to the WritePointRequest with field name 'value'

type WriteShardRequest 

type WriteShardRequest struct {
	// contains filtered or unexported fields
}

WriteShardRequest represents the a request to write a slice of points to a shard

func (*WriteShardRequest) AddPoint 

func (w *WriteShardRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)

func (*WriteShardRequest) AddPoints 

func (w *WriteShardRequest) AddPoints(points []tsdb.Point)

func (*WriteShardRequest) MarshalBinary 

func (w *WriteShardRequest) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*WriteShardRequest) Points 

func (w *WriteShardRequest) Points() []tsdb.Point

func (*WriteShardRequest) SetShardID 

func (w *WriteShardRequest) SetShardID(id uint64)

func (*WriteShardRequest) ShardID 

func (w *WriteShardRequest) ShardID() uint64

func (*WriteShardRequest) UnmarshalBinary 

func (w *WriteShardRequest) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates WritePointRequest from a binary format.

type WriteShardResponse 

type WriteShardResponse struct {
	// contains filtered or unexported fields
}

WriteShardResponse represents the response returned from a remote WriteShardRequest call

func (*WriteShardResponse) Code 

func (w *WriteShardResponse) Code() int

func (*WriteShardResponse) MarshalBinary 

func (w *WriteShardResponse) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*WriteShardResponse) Message 

func (w *WriteShardResponse) Message() string

func (*WriteShardResponse) SetCode 

func (w *WriteShardResponse) SetCode(code int)

func (*WriteShardResponse) SetMessage 

func (w *WriteShardResponse) SetMessage(message string)

func (*WriteShardResponse) UnmarshalBinary 

func (w *WriteShardResponse) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates WritePointRequest from a binary format.

Source Files

View all Source files

Directories

Path Synopsis
Package internal is a generated protocol buffer package.
 Package internal is a generated protocol buffer package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.