topo

package
v0.0.0-...-bc668d9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 3, 2015 License: BSD-3-Clause Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// According to docs, the tablet uid / (mysql server id) is uint32.
	// However, zero appears to be a sufficiently degenerate value to use
	// as a marker for not having a parent server id.
	// http://dev.mysql.com/doc/refman/5.1/en/replication-options.html
	NO_TABLET = 0

	// ReplicationLag is the key in the health map to indicate high
	// replication lag
	ReplicationLag = "replication_lag"

	// ReplicationLagHigh is the value in the health map to indicate high
	// replication lag
	ReplicationLagHigh = "high"
)
View Source
const (
	// idle -  no keyspace, shard or type assigned
	TYPE_IDLE = TabletType("idle")

	// primary copy of data
	TYPE_MASTER = TabletType("master")

	// a slaved copy of the data ready to be promoted to master
	TYPE_REPLICA = TabletType("replica")

	// a slaved copy of the data for olap load patterns.
	// too many aliases for olap - need to pick one
	TYPE_RDONLY = TabletType("rdonly")
	TYPE_BATCH  = TabletType("batch")

	// a slaved copy of the data ready, but not serving query traffic
	// could be a potential master.
	TYPE_SPARE = TabletType("spare")

	// a slaved copy of the data ready, but not serving query traffic
	// implies something abnormal about the setup - don't consider it
	// a potential master and don't worry about lag when reparenting.
	TYPE_EXPERIMENTAL = TabletType("experimental")

	// a slaved copy of the data intentionally lagged for pseudo backup
	TYPE_LAG = TabletType("lag")
	// when a reparent occurs, the tablet goes into lag_orphan state until
	// it can be reparented properly
	TYPE_LAG_ORPHAN = TabletType("lag_orphan")

	// a slaved copy of the data that was serving but is now applying
	// a schema change. Will go bak to its serving type after the
	// upgrade
	TYPE_SCHEMA_UPGRADE = TabletType("schema_apply")

	// a slaved copy of the data, but offline to queries other than backup
	// replication sql thread may be stopped
	TYPE_BACKUP = TabletType("backup")

	// a slaved copy of the data, where mysqld is *not* running,
	// and we are serving our data files to clone slaves
	// use 'vtctl Snapshot -server-mode ...' to get in this mode
	// use 'vtctl SnapshotSourceEnd ...' to get out of this mode
	TYPE_SNAPSHOT_SOURCE = TabletType("snapshot_source")

	// A tablet that has not been in the replication graph and is restoring
	// from a snapshot.  idle -> restore -> spare
	TYPE_RESTORE = TabletType("restore")

	// A tablet that is running a checker process. It is probably
	// lagging in replication.
	TYPE_CHECKER = TabletType("checker")

	// a machine with data that needs to be wiped
	TYPE_SCRAP = TabletType("scrap")
)
View Source
const (
	// STATE_READ_WRITE is the normal state for a master
	STATE_READ_WRITE = TabletState("ReadWrite")

	// STATE_READ_ONLY is the normal state for a slave, or temporarily a master.
	// Not to be confused with type, which implies a workload.
	STATE_READ_ONLY = TabletState("ReadOnly")
)
View Source
const (
	// DefaultPortName is the port named used by SrvEntries
	// if "" is given as the named port.
	DefaultPortName = "vt"
)
View Source
const SHARD_ZERO = "0"

This is the shard name for when the keyrange covers the entire space for unsharded database.

Variables

View Source
var (
	// ErrNodeExists is returned by functions to specify the
	// requested resource already exists.
	ErrNodeExists = errors.New("node already exists")

	// ErrNoNode is returned by functions to specify the requested
	// resource does not exist.
	ErrNoNode = errors.New("node doesn't exist")

	// ErrNotEmpty is returned by functions to specify a child of the
	// resource is still present and prevents the action from completing.
	ErrNotEmpty = errors.New("node not empty")

	// ErrTimeout is returned by functions that wait for a result
	// when the timeout value is reached.
	ErrTimeout = errors.New("deadline exceeded")

	// ErrInterrupted is returned by functions that wait for a result
	// when they are interrupted.
	ErrInterrupted = errors.New("interrupted")

	// ErrBadVersion is returned by an update function that
	// failed to update the data because the version was different
	ErrBadVersion = errors.New("bad node version")

	// ErrPartialResult is returned by a function that could only
	// get a subset of its results
	ErrPartialResult = errors.New("partial result")
)

AllTabletTypes lists all the possible tablet types

SlaveTabletTypes list all the types that are replication slaves

Functions

func CloseServers

func CloseServers()

CloseServers closes all registered Server.

func CreateShard

func CreateShard(ts Server, keyspace, shard string) error

CreateShard creates a new shard and tries to fill in the right information.

func CreateTablet

func CreateTablet(ts Server, tablet *Tablet) error

CreateTablet creates a new tablet and all associated paths for the replication graph.

func DeleteTabletReplicationData

func DeleteTabletReplicationData(ts Server, tablet *Tablet) error

DeleteTabletReplicationData deletes replication data.

func EndPointEquality

func EndPointEquality(left, right *EndPoint) bool

EndPointEquality returns true iff two EndPoint are representing the same data

func FindAllShardsInKeyspace

func FindAllShardsInKeyspace(ts Server, keyspace string) (map[string]*ShardInfo, error)

FindAllShardsInKeyspace reads and returns all the existing shards in a keyspace. It doesn't take any lock.

func FixShardReplication

func FixShardReplication(ts Server, logger logutil.Logger, cell, keyspace, shard string) error

FixShardReplication will fix the first problem it encounters within a ShardReplication object

func GetTabletMap

func GetTabletMap(ctx context.Context, ts Server, tabletAliases []TabletAlias) (map[TabletAlias]*TabletInfo, error)

GetTabletMap tries to read all the tablets in the provided list, and returns them all in a map. If error is ErrPartialResult, the results in the dictionary are incomplete, meaning some tablets couldn't be read.

func GetTabletMapForShard

func GetTabletMapForShard(ctx context.Context, ts Server, keyspace, shard string) (map[TabletAlias]*TabletInfo, error)

GetTabletMapForShard returns the tablets for a shard. It can return ErrPartialResult if it couldn't read all the cells, or all the individual tablets, in which case the map is valid, but partial.

func GetTabletMapForShardByCell

func GetTabletMapForShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) (map[TabletAlias]*TabletInfo, error)

GetTabletMapForShardByCell returns the tablets for a shard. It can return ErrPartialResult if it couldn't read all the cells, or all the individual tablets, in which case the map is valid, but partial.

func InCellList

func InCellList(cell string, cells []string) bool

InCellList returns true if the cell list is empty, or if the passed cell is in the cell list.

func IsInReplicationGraph

func IsInReplicationGraph(tt TabletType) bool

IsInReplicationGraph returns if this tablet appears in the replication graph Only IDLE and SCRAP are not in the replication graph. The other non-obvious types are BACKUP, SNAPSHOT_SOURCE, RESTORE and LAG_ORPHAN: these have had a master at some point (or were the master), so they are in the graph.

func IsInServingGraph

func IsInServingGraph(tt TabletType) bool

IsInServingGraph returns if a tablet appears in the serving graph

func IsRunningQueryService

func IsRunningQueryService(tt TabletType) bool

IsRunningQueryService returns if a tablet is running the query service

func IsRunningUpdateStream

func IsRunningUpdateStream(tt TabletType) bool

IsRunningUpdateStream returns if a tablet is running the update stream RPC service.

func IsSlaveType

func IsSlaveType(tt TabletType) bool

IsSlaveType returns if this type should be connected to a master db and actively replicating? MASTER is not obviously (only support one level replication graph) IDLE and SCRAP are not either BACKUP, RESTORE, LAG_ORPHAN, TYPE_CHECKER may or may not be, but we don't know for sure

func IsTrivialTypeChange

func IsTrivialTypeChange(oldTabletType, newTabletType TabletType) bool

IsTrivialTypeChange returns if this db type be trivially reassigned without changes to the replication graph

func IsTypeInList

func IsTypeInList(tabletType TabletType, types []TabletType) bool

IsTypeInList returns true if the given type is in the list. Use it with AllTabletType and SlaveTabletType for instance.

func IsValidTypeChange

func IsValidTypeChange(oldTabletType, newTabletType TabletType) bool

IsValidTypeChange returns if we should we allow this transition at all. Most transitions are allowed, but some don't make sense under any circumstances. If a transition could be forced, don't disallow it here.

func LookupVtName

func LookupVtName(ts Server, cell, keyspace, shard string, tabletType TabletType, namedPort string) ([]*net.SRV, error)

LookupVtName gets the list of EndPoints for a cell/keyspace/shard/tablet type and converts the list to net.SRV records

func MakeStringTypeList

func MakeStringTypeList(types []TabletType) []string

MakeStringTypeList returns a list of strings that match the input list.

func ParseKeyspaceShardString

func ParseKeyspaceShardString(param string) (string, string, error)

ParseKeyspaceShardString parse a "keyspace/shard" string and extract both keyspace and shard. It also returns empty keyspace and shard if input param looks like a old zk path

func ParseUID

func ParseUID(value string) (uint32, error)

ParseUID parses just the uid (a number)

func RegisterServer

func RegisterServer(name string, ts Server)

RegisterServer adds an implementation for a Server. If an implementation with that name already exists, panics. Call this in the 'init' function in your module.

func RemoveShardReplicationRecord

func RemoveShardReplicationRecord(ts Server, cell, keyspace, shard string, tabletAlias TabletAlias) error

RemoveShardReplicationRecord is a low level function to remove an entry from the ShardReplication object.

func ResolveKeyspaceWildcard

func ResolveKeyspaceWildcard(server WildcardBackend, param string) ([]string, error)

ResolveKeyspaceWildcard will resolve keyspace wildcards.

  • If the param is not a wildcard, it will just be returned (if the keyspace doesn't exist, it is still returned).
  • If the param is a wildcard, it will get all keyspaces and returns the ones which match the wildcard (which may be an empty list).

func SrvAddr

func SrvAddr(srv *net.SRV) string

SrvAddr prints a net.SRV

func SrvEntries

func SrvEntries(addrs *EndPoints, namedPort string) (srvs []*net.SRV, err error)

SrvEntries converts EndPoints to net.SRV for a given port. FIXME(msolomon) merge with zkns

func UpdateEndPoints

func UpdateEndPoints(ctx context.Context, ts Server, cell, keyspace, shard string, tabletType TabletType, addrs *EndPoints) error

UpdateEndPoints is a high level wrapper for TopoServer.UpdateEndPoints. It generates trace spans.

func UpdateKeyspace

func UpdateKeyspace(ts Server, ki *KeyspaceInfo) error

UpdateKeyspace updates the keyspace data, with the right version

func UpdateShard

func UpdateShard(ctx context.Context, ts Server, si *ShardInfo) error

UpdateShard updates the shard data, with the right version

func UpdateShardReplicationRecord

func UpdateShardReplicationRecord(ctx context.Context, ts Server, keyspace, shard string, tabletAlias TabletAlias) error

UpdateShardReplicationRecord is a low level function to add / update an entry to the ShardReplication object.

func UpdateTablet

func UpdateTablet(ctx context.Context, ts Server, tablet *TabletInfo) error

UpdateTablet updates the tablet data only - not associated replication paths.

func UpdateTabletFields

func UpdateTabletFields(ctx context.Context, ts Server, alias TabletAlias, update func(*Tablet) error) error

UpdateTabletFields is a high level wrapper for TopoServer.UpdateTabletFields that generates trace spans.

func UpdateTabletReplicationData

func UpdateTabletReplicationData(ctx context.Context, ts Server, tablet *Tablet) error

UpdateTabletReplicationData creates or updates the replication graph data for a tablet

func Validate

func Validate(ts Server, tabletAlias TabletAlias) error

Validate makes sure a tablet is represented correctly in the topology server.

func ValidateShardName

func ValidateShardName(shard string) (string, key.KeyRange, error)

ValidateShardName takes a shard name and sanitizes it, and also returns the KeyRange.

Types

type EndPoint

type EndPoint struct {
	Uid          uint32            `json:"uid"` // Keep track of which tablet this corresponds to.
	Host         string            `json:"host"`
	NamedPortMap map[string]int    `json:"named_port_map"`
	Health       map[string]string `json:"health"`
}

EndPoint describes a tablet (maybe composed of multiple processes) listening on one or more named ports, and its health. Clients use this record to connect to a tablet.

func NewEndPoint

func NewEndPoint(uid uint32, host string) *EndPoint

NewEndPoint returns a new empty EndPoint

type EndPoints

type EndPoints struct {
	Entries []EndPoint `json:"entries"`
}

EndPoints is a list of EndPoint objects, all of the same type.

func NewEndPoints

func NewEndPoints() *EndPoints

NewEndPoints creates a EndPoints with a pre-allocated slice for Entries.

type GetEndPointsArgs

type GetEndPointsArgs struct {
	Cell       string
	Keyspace   string
	Shard      string
	TabletType TabletType
}

GetEndPointsArgs is the parameters for TopoReader.GetEndPoints

func (*GetEndPointsArgs) MarshalBson

func (getEndPointsArgs *GetEndPointsArgs) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes GetEndPointsArgs.

func (*GetEndPointsArgs) UnmarshalBson

func (getEndPointsArgs *GetEndPointsArgs) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into GetEndPointsArgs.

type GetSrvKeyspaceArgs

type GetSrvKeyspaceArgs struct {
	Cell     string
	Keyspace string
}

GetSrvKeyspaceArgs is the parameters for TopoReader.GetSrvKeyspace

func (*GetSrvKeyspaceArgs) MarshalBson

func (getSrvKeyspaceArgs *GetSrvKeyspaceArgs) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes GetSrvKeyspaceArgs.

func (*GetSrvKeyspaceArgs) UnmarshalBson

func (getSrvKeyspaceArgs *GetSrvKeyspaceArgs) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into GetSrvKeyspaceArgs.

type GetSrvKeyspaceNamesArgs

type GetSrvKeyspaceNamesArgs struct {
	Cell string
}

GetSrvKeyspaceNamesArgs is the parameters for TopoReader.GetSrvKeyspaceNames

func (*GetSrvKeyspaceNamesArgs) MarshalBson

func (getSrvKeyspaceNamesArgs *GetSrvKeyspaceNamesArgs) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes GetSrvKeyspaceNamesArgs.

func (*GetSrvKeyspaceNamesArgs) UnmarshalBson

func (getSrvKeyspaceNamesArgs *GetSrvKeyspaceNamesArgs) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into GetSrvKeyspaceNamesArgs.

type GetSrvShardArgs

type GetSrvShardArgs struct {
	Cell     string
	Keyspace string
	Shard    string
}

GetSrvShardArgs is the parameters for TopoReader.GetSrvShard

func (*GetSrvShardArgs) MarshalBson

func (getSrvShardArgs *GetSrvShardArgs) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes GetSrvShardArgs.

func (*GetSrvShardArgs) UnmarshalBson

func (getSrvShardArgs *GetSrvShardArgs) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into GetSrvShardArgs.

type Keyspace

type Keyspace struct {
	// name of the column used for sharding
	// empty if the keyspace is not sharded
	ShardingColumnName string

	// type of the column used for sharding
	// KIT_UNSET if the keyspace is not sharded
	ShardingColumnType key.KeyspaceIdType

	// ServedFromMap will redirect the appropriate traffic to
	// another keyspace
	ServedFromMap map[TabletType]*KeyspaceServedFrom

	// Number of shards to use for batch job / mapreduce jobs
	// that need to split a given keyspace into multiple shards.
	// The value N used should be big enough that all possible shards
	// cover 1/Nth of the entire space or more.
	// It is usually the number of shards in the system. If a keyspace
	// is being resharded from M to P shards, it should be max(M, P).
	// That way we can guarantee a query that is targeted to 1/N of the
	// keyspace will land on just one shard.
	SplitShardCount int32
}

Keyspace is the data structure that has data about the Keyspaces in the topology. Most fields are optional.

type KeyspaceInfo

type KeyspaceInfo struct {
	*Keyspace
	// contains filtered or unexported fields
}

KeyspaceInfo is a meta struct that contains metadata to give the data more context and convenience. This is the main way we interact with a keyspace.

func NewKeyspaceInfo

func NewKeyspaceInfo(keyspace string, value *Keyspace, version int64) *KeyspaceInfo

NewKeyspaceInfo returns a KeyspaceInfo basing on keyspace with the keyspace. This function should be only used by Server implementations.

func (*KeyspaceInfo) CheckServedFromMigration

func (ki *KeyspaceInfo) CheckServedFromMigration(tabletType TabletType, cells []string, keyspace string, remove bool) error

CheckServedFromMigration makes sure a requested migration is safe

func (*KeyspaceInfo) ComputeCellServedFrom

func (ki *KeyspaceInfo) ComputeCellServedFrom(cell string) map[TabletType]string

ComputeCellServedFrom returns the ServedFrom map for a cell

func (*KeyspaceInfo) KeyspaceName

func (ki *KeyspaceInfo) KeyspaceName() string

KeyspaceName returns the keyspace name

func (*KeyspaceInfo) UpdateServedFromMap

func (ki *KeyspaceInfo) UpdateServedFromMap(tabletType TabletType, cells []string, keyspace string, remove bool, allCells []string) error

UpdateServedFromMap handles ServedFromMap. It can add or remove records, cells, ...

func (*KeyspaceInfo) Version

func (ki *KeyspaceInfo) Version() int64

Version returns the keyspace version from last time it was read or updated.

type KeyspacePartition

type KeyspacePartition struct {
	// List of non-overlapping continuous shards sorted by range.
	Shards []SrvShard
}

KeyspacePartition represents a continuous set of shards to serve an entire data set.

func (*KeyspacePartition) HasShard

func (kp *KeyspacePartition) HasShard(name string) bool

HasShard returns true if this KeyspacePartition has the shard with the given name in it.

func (*KeyspacePartition) MarshalBson

func (keyspacePartition *KeyspacePartition) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes KeyspacePartition.

func (*KeyspacePartition) UnmarshalBson

func (keyspacePartition *KeyspacePartition) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into KeyspacePartition.

type KeyspaceServedFrom

type KeyspaceServedFrom struct {
	// who is targeted
	Cells []string // nil means all cells

	// where to redirect
	Keyspace string
}

KeyspaceServedFrom is a per-cell record to redirect traffic to another keyspace. Used for vertical splits.

type KeyspaceShard

type KeyspaceShard struct {
	Keyspace string
	Shard    string
}

KeyspaceShard is a type used by ResolveShardWildcard

func ResolveShardWildcard

func ResolveShardWildcard(server WildcardBackend, param string) ([]KeyspaceShard, error)

ResolveShardWildcard will resolve shard wildcards. Both keyspace and shard names can use wildcard. Errors talking to the topology server are returned. ErrNoNode is ignored if it's the result of resolving a wildcard. Examples:

  • */* returns all keyspace/shard pairs, or empty list if none.
  • user/* returns all shards in user keyspace (or error if user keyspace doesn't exist)
  • us*/* returns all shards in all keyspaces that start with 'us'. If no such keyspace exists, list is empty (it is not an error).
type ReplicationLink struct {
	TabletAlias TabletAlias
}

ReplicationLink describes a tablet that is linked in a shard. We will add a record for all tablets in a shard.

type Schemafier

type Schemafier interface {
	SaveVSchema(string) error
	GetVSchema() (string, error)
}

Schemafier is a temporary interface for supporting vschema reads and writes. It will eventually be merged into Server.

type Server

type Server interface {
	// topo.Server management interface.
	Close()

	// GetKnownCells returns the list of known cells running our processes.
	// It is possible to find all tablets in the entire system
	// by then calling GetTabletsByCell on every cell, for instance.
	// They shall be sorted.
	GetKnownCells() ([]string, error)

	// CreateKeyspace creates the given keyspace, assuming it doesn't exist
	// yet. Can return ErrNodeExists if it already exists.
	CreateKeyspace(keyspace string, value *Keyspace) error

	// UpdateKeyspace updates the keyspace information
	// pointed at by ki.keyspace to the *ki value.
	// This will only be called with a lock on the keyspace.
	// Can return ErrNoNode if the keyspace doesn't exist yet,
	// or ErrBadVersion if the version has changed.
	//
	// Do not use directly, but instead use topo.UpdateKeyspace.
	UpdateKeyspace(ki *KeyspaceInfo, existingVersion int64) (newVersion int64, err error)

	// GetKeyspace reads a keyspace and returns it.
	// Can return ErrNoNode
	GetKeyspace(keyspace string) (*KeyspaceInfo, error)

	// GetKeyspaces returns the known keyspace names. They shall be sorted.
	GetKeyspaces() ([]string, error)

	// DeleteKeyspaceShards deletes all the shards in a keyspace.
	// Use with caution.
	DeleteKeyspaceShards(keyspace string) error

	// CreateShard creates an empty shard, assuming it doesn't exist
	// yet. The contents of the shard will be a new Shard{} object,
	// with KeyRange populated by the result of ValidateShardName().
	// Can return ErrNodeExists if it already exists.
	CreateShard(keyspace, shard string, value *Shard) error

	// UpdateShard updates the shard information
	// pointed at by si.keyspace / si.shard to the *si value.
	// This will only be called with a lock on the shard.
	// Can return ErrNoNode if the shard doesn't exist yet,
	// or ErrBadVersion if the version has changed.
	//
	// Do not use directly, but instead use topo.UpdateShard.
	UpdateShard(si *ShardInfo, existingVersion int64) (newVersion int64, err error)

	// ValidateShard performs routine checks on the shard.
	ValidateShard(keyspace, shard string) error

	// GetShard reads a shard and returns it.
	// Can return ErrNoNode
	GetShard(keyspace, shard string) (*ShardInfo, error)

	// GetShardNames returns the known shards in a keyspace.
	// Can return ErrNoNode if the keyspace wasn't created,
	// or if DeleteKeyspaceShards was called. They shall be sorted.
	GetShardNames(keyspace string) ([]string, error)

	// DeleteShard deletes the provided shard.
	// Can return ErrNoNode if the shard doesn't exist.
	DeleteShard(keyspace, shard string) error

	// CreateTablet creates the given tablet, assuming it doesn't exist
	// yet. It does *not* create the tablet replication paths.
	// Can return ErrNodeExists if it already exists.
	CreateTablet(tablet *Tablet) error

	// UpdateTablet updates a given tablet. The version is used
	// for atomic updates. UpdateTablet will return ErrNoNode if
	// the tablet doesn't exist and ErrBadVersion if the version
	// has changed.
	//
	// Do not use directly, but instead use topo.UpdateTablet.
	UpdateTablet(tablet *TabletInfo, existingVersion int64) (newVersion int64, err error)

	// UpdateTabletFields updates the current tablet record
	// with new values, independently of the version
	// Can return ErrNoNode if the tablet doesn't exist.
	UpdateTabletFields(tabletAlias TabletAlias, update func(*Tablet) error) error

	// DeleteTablet removes a tablet from the system.
	// We assume no RPC is currently running to it.
	// TODO(alainjobart) verify this assumption, link with RPC code.
	// Can return ErrNoNode if the tablet doesn't exist.
	DeleteTablet(alias TabletAlias) error

	// GetTablet returns the tablet data (includes the current version).
	// Can return ErrNoNode if the tablet doesn't exist.
	GetTablet(alias TabletAlias) (*TabletInfo, error)

	// GetTabletsByCell returns all the tablets in the given cell.
	// Can return ErrNoNode if no tablet was ever created in that cell.
	GetTabletsByCell(cell string) ([]TabletAlias, error)

	// UpdateShardReplicationFields updates the current
	// ShardReplication record with new values. If the
	// ShardReplication object does not exist, an empty one will
	// be passed to the update function. All necessary directories
	// need to be created by this method, if applicable.
	UpdateShardReplicationFields(cell, keyspace, shard string, update func(*ShardReplication) error) error

	// GetShardReplication returns the replication data.
	// Can return ErrNoNode if the object doesn't exist.
	GetShardReplication(cell, keyspace, shard string) (*ShardReplicationInfo, error)

	// DeleteShardReplication deletes the replication data.
	// Can return ErrNoNode if the object doesn't exist.
	DeleteShardReplication(cell, keyspace, shard string) error

	// LockSrvShardForAction locks the serving shard in order to
	// perform the action described by contents. It will wait for
	// the lock until at most ctx.Done(). The wait can be interrupted
	// by cancelling the context. It returns the lock path.
	//
	// Can return ErrTimeout or ErrInterrupted.
	LockSrvShardForAction(ctx context.Context, cell, keyspace, shard, contents string) (string, error)

	// UnlockSrvShardForAction unlocks a serving shard.
	UnlockSrvShardForAction(cell, keyspace, shard, lockPath, results string) error

	// GetSrvTabletTypesPerShard returns the existing serving types
	// for a shard.
	// Can return ErrNoNode.
	GetSrvTabletTypesPerShard(cell, keyspace, shard string) ([]TabletType, error)

	// UpdateEndPoints updates the serving records for a cell,
	// keyspace, shard, tabletType.
	UpdateEndPoints(cell, keyspace, shard string, tabletType TabletType, addrs *EndPoints) error

	// GetEndPoints returns the EndPoints list of serving addresses
	// for a TabletType inside a shard.
	// Can return ErrNoNode.
	GetEndPoints(cell, keyspace, shard string, tabletType TabletType) (*EndPoints, error)

	// DeleteEndPoints deletes the serving records for a cell,
	// keyspace, shard, tabletType.
	// Can return ErrNoNode.
	DeleteEndPoints(cell, keyspace, shard string, tabletType TabletType) error

	// WatchEndPoints returns a channel that receives notifications
	// every time EndPoints for the given type changes.
	// It should receive a notification with the initial value fairly
	// quickly after this is set. A value of nil means the Endpoints
	// object doesn't exist or is empty. To stop watching this
	// EndPoints object, close the stopWatching channel.
	// If the underlying topo.Server encounters an error watching the node,
	// it should retry on a regular basis until it can succeed.
	// The initial error returned by this method is meant to catch
	// the obvious bad cases (invalid cell, invalid tabletType, ...)
	// that are never going to work. Mutiple notifications with the
	// same contents may be sent (for instance when the serving graph
	// is rebuilt, but the content hasn't changed).
	WatchEndPoints(cell, keyspace, shard string, tabletType TabletType) (notifications <-chan *EndPoints, stopWatching chan<- struct{}, err error)

	// UpdateSrvShard updates the serving records for a cell,
	// keyspace, shard.
	UpdateSrvShard(cell, keyspace, shard string, srvShard *SrvShard) error

	// GetSrvShard reads a SrvShard record.
	// Can return ErrNoNode.
	GetSrvShard(cell, keyspace, shard string) (*SrvShard, error)

	// DeleteSrvShard deletes a SrvShard record.
	// Can return ErrNoNode.
	DeleteSrvShard(cell, keyspace, shard string) error

	// UpdateSrvKeyspace updates the serving records for a cell, keyspace.
	UpdateSrvKeyspace(cell, keyspace string, srvKeyspace *SrvKeyspace) error

	// GetSrvKeyspace reads a SrvKeyspace record.
	// Can return ErrNoNode.
	GetSrvKeyspace(cell, keyspace string) (*SrvKeyspace, error)

	// GetSrvKeyspaceNames returns the list of visible Keyspaces
	// in this cell. They shall be sorted.
	GetSrvKeyspaceNames(cell string) ([]string, error)

	// UpdateTabletEndpoint updates a single tablet record in the
	// already computed serving graph. The update has to be somewhat
	// atomic, so it requires Server intrisic knowledge.
	// If the node doesn't exist, it is not updated, this is not an error.
	UpdateTabletEndpoint(cell, keyspace, shard string, tabletType TabletType, addr *EndPoint) error

	// LockKeyspaceForAction locks the keyspace in order to
	// perform the action described by contents. It will wait for
	// the lock until at most ctx.Done(). The wait can be interrupted
	// by cancelling the context. It returns the lock path.
	//
	// Can return ErrTimeout or ErrInterrupted
	LockKeyspaceForAction(ctx context.Context, keyspace, contents string) (string, error)

	// UnlockKeyspaceForAction unlocks a keyspace.
	UnlockKeyspaceForAction(keyspace, lockPath, results string) error

	// LockShardForAction locks the shard in order to
	// perform the action described by contents. It will wait for
	// the lock until at most ctx.Done(). The wait can be interrupted
	// by cancelling the context. It returns the lock path.
	//
	// Can return ErrTimeout or ErrInterrupted
	LockShardForAction(ctx context.Context, keyspace, shard, contents string) (string, error)

	// UnlockShardForAction unlocks a shard.
	UnlockShardForAction(keyspace, shard, lockPath, results string) error
}

Server is the interface used to talk to a persistent backend storage server and locking service.

Zookeeper is a good example of this, and zktopo contains the implementation for this using zookeeper.

Inside Google, we use Chubby.

func GetServer

func GetServer() Server

GetServer returns 'our' Server, going down this list:

  • If only one is registered, that's the one.
  • If more than one are registered, use the 'topo_implementation' flag (which defaults to zookeeper).
  • Then panics.

func GetServerByName

func GetServerByName(name string) Server

GetServerByName returns a specific Server by name, or nil.

type Shard

type Shard struct {
	// There can be only at most one master, but there may be none. (0)
	MasterAlias TabletAlias

	// This must match the shard name based on our other conventions, but
	// helpful to have it decomposed here.
	KeyRange key.KeyRange

	// ServedTypesMap is a map of all the tablet types this shard
	// will serve, to the cells that serve this type. This is
	// usually used with overlapping shards during data shuffles
	// like shard splitting. Note the master record will always
	// list all the cells.
	ServedTypesMap map[TabletType]*ShardServedType

	// SourceShards is the list of shards we're replicating from,
	// using filtered replication.
	SourceShards []SourceShard

	// Cells is the list of cells that have tablets for this shard.
	// It is populated at InitTablet time when a tablet is added
	// in a cell that is not in the list yet.
	Cells []string

	// TabletControlMap is a map of TabletControl to apply specific
	// configurations to tablets by type.
	TabletControlMap map[TabletType]*TabletControl
}

Shard is a pure data struct for information stored in topology server. This node is used to present a controlled view of the shard, unaware of every management action. It also contains configuration data for a shard.

func (*Shard) HasCell

func (shard *Shard) HasCell(cell string) bool

HasCell returns true if the cell is listed in the Cells for the shard.

type ShardInfo

type ShardInfo struct {
	*Shard
	// contains filtered or unexported fields
}

ShardInfo is a meta struct that contains metadata to give the data more context and convenience. This is the main way we interact with a shard.

func GetShard

func GetShard(ctx context.Context, ts Server, keyspace, shard string) (*ShardInfo, error)

GetShard is a high level function to read shard data. It generates trace spans.

func NewShardInfo

func NewShardInfo(keyspace, shard string, value *Shard, version int64) *ShardInfo

NewShardInfo returns a ShardInfo basing on shard with the keyspace / shard. This function should be only used by Server implementations.

func UpdateShardFields

func UpdateShardFields(ctx context.Context, ts Server, keyspace, shard string, update func(*Shard) error) (*ShardInfo, error)

UpdateShardFields is a high level helper to read a shard record, call an update function on it, and then write it back. If the write fails due to a version mismatch, it will re-read the record and retry the update. If the update succeeds, it returns the updated ShardInfo.

func (*ShardInfo) CheckServedTypesMigration

func (si *ShardInfo) CheckServedTypesMigration(tabletType TabletType, cells []string, remove bool) error

CheckServedTypesMigration makes sure the provided migration is possible

func (*ShardInfo) GetServedTypesPerCell

func (si *ShardInfo) GetServedTypesPerCell(cell string) []TabletType

GetServedTypesPerCell returns the list of types this shard is serving in the provided cell.

func (*ShardInfo) Keyspace

func (si *ShardInfo) Keyspace() string

Keyspace returns the keyspace a shard belongs to

func (*ShardInfo) ShardName

func (si *ShardInfo) ShardName() string

ShardName returns the shard name for a shard

func (*ShardInfo) UpdateDisableQueryService

func (si *ShardInfo) UpdateDisableQueryService(tabletType TabletType, cells []string, disableQueryService bool) error

UpdateDisableQueryService will make sure the disableQueryService is set appropriately in the shard record. Note we don't support a lot of the corner cases:

  • we don't support DisableQueryService at the same time as BlacklistedTables, because it's not used in the same context (vertical vs horizontal sharding)

func (*ShardInfo) UpdateServedTypesMap

func (si *ShardInfo) UpdateServedTypesMap(tabletType TabletType, cells []string, remove bool) error

UpdateServedTypesMap handles ServedTypesMap. It can add or remove records, cells, ...

func (*ShardInfo) UpdateSourceBlacklistedTables

func (si *ShardInfo) UpdateSourceBlacklistedTables(tabletType TabletType, cells []string, remove bool, tables []string) error

UpdateSourceBlacklistedTables will add or remove the listed tables in the shard record's TabletControl structures. Note we don't support a lot of the corner cases:

  • only support one table list per shard. If we encounter a different table list that the provided one, we error out.
  • we don't support DisableQueryService at the same time as BlacklistedTables, because it's not used in the same context (vertical vs horizontal sharding)

func (*ShardInfo) Version

func (si *ShardInfo) Version() int64

Version returns the shard version from last time it was read or updated.

type ShardReplication

type ShardReplication struct {
	// Note there can be only one ReplicationLink in this array
	// for a given Tablet
	ReplicationLinks []ReplicationLink
}

ShardReplication describes all the tablets for a shard whithin a cell.

func (sr *ShardReplication) GetReplicationLink(tabletAlias TabletAlias) (ReplicationLink, error)

GetReplicationLink find a link for a given tablet.

type ShardReplicationInfo

type ShardReplicationInfo struct {
	*ShardReplication
	// contains filtered or unexported fields
}

ShardReplicationInfo is the companion structure for ShardReplication.

func NewShardReplicationInfo

func NewShardReplicationInfo(sr *ShardReplication, cell, keyspace, shard string) *ShardReplicationInfo

NewShardReplicationInfo is for topo.Server implementations to create the structure

func (*ShardReplicationInfo) Cell

func (sri *ShardReplicationInfo) Cell() string

Cell returns the cell for a ShardReplicationInfo

func (*ShardReplicationInfo) Keyspace

func (sri *ShardReplicationInfo) Keyspace() string

Keyspace returns the keyspace for a ShardReplicationInfo

func (*ShardReplicationInfo) Shard

func (sri *ShardReplicationInfo) Shard() string

Shard returns the shard for a ShardReplicationInfo

type ShardServedType

type ShardServedType struct {
	Cells []string // nil means all cells
}

ShardServedType describes the cells where the given shard is serving.

type SourceShard

type SourceShard struct {
	// Uid is the unique ID for this SourceShard object.
	// It is for instance used as a unique index in blp_checkpoint
	// when storing the position. It should be unique whithin a
	// destination Shard, but not globally unique.
	Uid uint32

	// the source keyspace
	Keyspace string

	// the source shard
	Shard string

	// The source shard keyrange
	// If partial, len(Tables) has to be zero
	KeyRange key.KeyRange

	// The source table list to replicate
	// If non-empty, KeyRange must not be partial (must be KeyRange{})
	Tables []string
}

SourceShard represents a data source for filtered replication across shards. When this is used in a destination shard, the master of that shard will run filtered replication.

func (*SourceShard) AsHTML

func (source *SourceShard) AsHTML() template.HTML

AsHTML returns a HTML version of the object.

func (*SourceShard) String

func (source *SourceShard) String() string

String returns a printable view of a SourceShard.

type SrvKeyspace

type SrvKeyspace struct {
	// Shards to use per type, only contains complete partitions.
	Partitions map[TabletType]*KeyspacePartition

	// List of available tablet types for this keyspace in this cell.
	// May not have a server for every shard, but we have some.
	TabletTypes []TabletType

	// Copied from Keyspace
	ShardingColumnName string
	ShardingColumnType key.KeyspaceIdType
	ServedFrom         map[TabletType]string
	SplitShardCount    int32
	// contains filtered or unexported fields
}

A distilled serving copy of keyspace detail stored in the local cell for fast access. Derived from the global keyspace, shards and local details. In zk, it is in /zk/<cell>/vt/ns/<keyspace>

func NewSrvKeyspace

func NewSrvKeyspace(version int64) *SrvKeyspace

NewSrvKeyspace returns an empty SrvKeyspace with the given version.

func (*SrvKeyspace) MarshalBson

func (srvKeyspace *SrvKeyspace) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes SrvKeyspace.

func (*SrvKeyspace) UnmarshalBson

func (srvKeyspace *SrvKeyspace) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into SrvKeyspace.

type SrvKeyspaceNames

type SrvKeyspaceNames struct {
	Entries []string
}

SrvKeyspaceNames is the response for TopoReader.GetSrvKeyspaceNames

func (*SrvKeyspaceNames) MarshalBson

func (srvKeyspaceNames *SrvKeyspaceNames) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes SrvKeyspaceNames.

func (*SrvKeyspaceNames) UnmarshalBson

func (srvKeyspaceNames *SrvKeyspaceNames) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into SrvKeyspaceNames.

type SrvShard

type SrvShard struct {
	// Copied / inferred from Shard
	Name        string
	KeyRange    key.KeyRange
	ServedTypes []TabletType

	// MasterCell indicates the cell that master tablet resides
	MasterCell string

	// TabletTypes represents the list of types we have serving tablets
	// for, in this cell only.
	TabletTypes []TabletType
	// contains filtered or unexported fields
}

SrvShard contains a roll-up of the shard in the local namespace. In zk, it is under /zk/<cell>/vt/ns/<keyspace>/<shard>

func NewSrvShard

func NewSrvShard(version int64) *SrvShard

NewSrvShard returns an empty SrvShard with the given version.

func (*SrvShard) MarshalBson

func (srvShard *SrvShard) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes SrvShard.

func (*SrvShard) ShardName

func (ss *SrvShard) ShardName() string

ShardName returns the name of a shard.

func (*SrvShard) UnmarshalBson

func (srvShard *SrvShard) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into SrvShard.

type SrvShardArray

type SrvShardArray []SrvShard

SrvShardArray is used for sorting SrvShard arrays

func (SrvShardArray) Len

func (sa SrvShardArray) Len() int

Len implements sort.Interface

func (SrvShardArray) Less

func (sa SrvShardArray) Less(i, j int) bool

Len implements sort.Interface

func (SrvShardArray) Sort

func (sa SrvShardArray) Sort()

func (SrvShardArray) Swap

func (sa SrvShardArray) Swap(i, j int)

Len implements sort.Interface

type Tablet

type Tablet struct {
	// What is this tablet?
	Alias TabletAlias

	// Location of the tablet
	Hostname string
	IPAddr   string

	// Named port names. Currently supported ports: vt, vts,
	// mysql.
	Portmap map[string]int

	// Tags contain freeform information about the tablet.
	Tags map[string]string

	// Health tracks how healthy the tablet is. Clients may decide
	// to use this information to make educated decisions on which
	// tablet to connect to.
	Health map[string]string

	// Information about the tablet inside a keyspace/shard
	Keyspace string
	Shard    string
	Type     TabletType

	// Is the tablet read-only?
	State TabletState

	// Normally the database name is implied by "vt_" + keyspace. I
	// really want to remove this but there are some databases that are
	// hard to rename.
	DbNameOverride string
	KeyRange       key.KeyRange
}

Tablet is a pure data struct for information serialized into json and stored into topo.Server

func (*Tablet) Addr

func (tablet *Tablet) Addr() string

Addr returns hostname:vt port.

func (*Tablet) Complete

func (tablet *Tablet) Complete() error

Complete validates and normalizes the tablet. If the shard name contains a '-' it is going to try to infer the keyrange from it.

func (*Tablet) DbName

func (tablet *Tablet) DbName() string

DbName is usually implied by keyspace. Having the shard information in the database name complicates mysql replication.

func (*Tablet) EndPoint

func (tablet *Tablet) EndPoint() (*EndPoint, error)

EndPoint returns an EndPoint associated with the tablet record

func (*Tablet) IsAssigned

func (tablet *Tablet) IsAssigned() bool

IsAssigned returns if this tablet ever assigned data? A "scrap" node will show up as assigned even though its data cannot be used for serving.

func (*Tablet) IsHealthEqual

func (tablet *Tablet) IsHealthEqual(health map[string]string) bool

IsHealthEqual compares the tablet's health with the passed one, and returns true if they're equivalent.

func (*Tablet) IsInReplicationGraph

func (tablet *Tablet) IsInReplicationGraph() bool

IsInReplicationGraph returns if this tablet is in the replication graph.

func (*Tablet) IsInServingGraph

func (tablet *Tablet) IsInServingGraph() bool

IsInServingGraph returns if this tablet is in the serving graph

func (*Tablet) IsRunningQueryService

func (tablet *Tablet) IsRunningQueryService() bool

IsRunningQueryService returns if this tablet should be running the query service.

func (*Tablet) IsSlaveType

func (tablet *Tablet) IsSlaveType() bool

IsSlaveType returns if this tablet's type is a slave

func (*Tablet) JSON

func (tablet *Tablet) JSON() string

JSON returns a json verison of the tablet.

func (*Tablet) MysqlAddr

func (tablet *Tablet) MysqlAddr() string

MysqlAddr returns hostname:mysql port.

func (*Tablet) MysqlIPAddr

func (tablet *Tablet) MysqlIPAddr() string

MysqlIPAddr returns ip:mysql port.

func (*Tablet) String

func (tablet *Tablet) String() string

String returns a string describing the tablet.

func (*Tablet) ValidatePortmap

func (tablet *Tablet) ValidatePortmap() error

ValidatePortmap returns an error if the tablet's portmap doesn't contain all the necessary ports for the tablet to be fully operational. We only care about vt port now, as mysql may not even be running.

type TabletAlias

type TabletAlias struct {
	Cell string
	Uid  uint32
}

TabletAlias is the minimum required information to locate a tablet.

Tablets are really globally unique, but crawling every cell to find out where it lives is time consuming and expensive. This is only needed during complex operations. Tablet cell assignments don't change that often, thus using a TabletAlias is efficient.

func FindAllTabletAliasesInShard

func FindAllTabletAliasesInShard(ctx context.Context, ts Server, keyspace, shard string) ([]TabletAlias, error)

FindAllTabletAliasesInShard uses the replication graph to find all the tablet aliases in the given shard. It can return ErrPartialResult if some cells were not fetched, in which case the result only contains the cells that were fetched.

func FindAllTabletAliasesInShardByCell

func FindAllTabletAliasesInShardByCell(ctx context.Context, ts Server, keyspace, shard string, cells []string) ([]TabletAlias, error)

FindAllTabletAliasesInShardByCell uses the replication graph to find all the tablet aliases in the given shard. It can return ErrPartialResult if some cells were not fetched, in which case the result only contains the cells that were fetched.

func ParseTabletAliasString

func ParseTabletAliasString(aliasStr string) (result TabletAlias, err error)

ParseTabletAliasString returns a TabletAlias for the input string, of the form <cell>-<uid>

func (TabletAlias) IsZero

func (ta TabletAlias) IsZero() bool

IsZero returns true iff cell and uid are empty

func (TabletAlias) String

func (ta TabletAlias) String() string

String formats a TabletAlias

func (TabletAlias) TabletUIDStr

func (ta TabletAlias) TabletUIDStr() string

TabletUIDStr returns a string version of the uid

type TabletAliasList

type TabletAliasList []TabletAlias

TabletAliasList is used mainly for sorting

func (TabletAliasList) Len

func (tal TabletAliasList) Len() int

Len is part of sort.Interface

func (TabletAliasList) Less

func (tal TabletAliasList) Less(i, j int) bool

Less is part of sort.Interface

func (TabletAliasList) Swap

func (tal TabletAliasList) Swap(i, j int)

Swap is part of sort.Interface

type TabletControl

type TabletControl struct {
	// How to match the tablets
	Cells []string // nil means all cells

	// What specific action to take
	DisableQueryService bool
	BlacklistedTables   []string // only used if DisableQueryService==false
}

TabletControl describes the parameters used by the vttablet processes to know what specific configurations they should be using.

type TabletInfo

type TabletInfo struct {
	*Tablet
	// contains filtered or unexported fields
}

TabletInfo is the container for a Tablet, read from the topology server.

func GetTablet

func GetTablet(ctx context.Context, ts Server, alias TabletAlias) (*TabletInfo, error)

GetTablet is a high level function to read tablet data. It generates trace spans.

func NewTabletInfo

func NewTabletInfo(tablet *Tablet, version int64) *TabletInfo

NewTabletInfo returns a TabletInfo basing on tablet with the version set. This function should be only used by Server implementations.

func (*TabletInfo) Version

func (ti *TabletInfo) Version() int64

Version returns the version of this tablet from last time it was read or updated.

type TabletState

type TabletState string

TabletState describe if the tablet is read-only or read-write.

type TabletType

type TabletType string

TabletType is the main type for a tablet. It has an implication on: - the replication graph - the services run by vttablet on a tablet - the uptime expectancy

func (TabletType) IsSlaveType

func (tt TabletType) IsSlaveType() bool

IsSlaveType returns true iff the type is a mysql replication slave.

func (TabletType) MarshalBson

func (tabletType TabletType) MarshalBson(buf *bytes2.ChunkedWriter, key string)

MarshalBson bson-encodes TabletType.

func (*TabletType) UnmarshalBson

func (tabletType *TabletType) UnmarshalBson(buf *bytes.Buffer, kind byte)

UnmarshalBson bson-decodes into TabletType.

type TopoReader

type TopoReader interface {
	// GetSrvKeyspaces returns the names of all the keyspaces in
	// the topology for the cell.
	GetSrvKeyspaceNames(context.Context, *GetSrvKeyspaceNamesArgs, *SrvKeyspaceNames) error

	// GetSrvKeyspace returns information about a keyspace in a
	// particular cell (as specified by the GetSrvKeyspaceArgs).
	GetSrvKeyspace(context.Context, *GetSrvKeyspaceArgs, *SrvKeyspace) error

	// GetSrvShard returns information about a shard in a
	// particular cell and keyspace (as specified by the GetSrvShardArgs).
	GetSrvShard(context.Context, *GetSrvShardArgs, *SrvShard) error

	// GetEndPoints returns addresses for a tablet type in a shard
	// in a keyspace (as specified in GetEndPointsArgs).
	GetEndPoints(context.Context, *GetEndPointsArgs, *EndPoints) error
}

TopoReader returns read only information about the topology.

type WildcardBackend

type WildcardBackend interface {
	// GetKeyspaces returns the known keyspaces. They shall be sorted.
	GetKeyspaces() ([]string, error)

	// GetShard reads a shard and returns it.
	// Can return ErrNoNode
	GetShard(keyspace, shard string) (*ShardInfo, error)

	// GetShardNames returns the known shards in a keyspace.
	// Can return ErrNoNode
	GetShardNames(keyspace string) ([]string, error)
}

WildcardBackend is a subset of Server for the methods used by the wildcard code. This lets us test with a very simple fake topo server.

Directories

Path Synopsis
Package events defines the structures used for events relating directly to individual topology elements, like keyspaces, shards, and tablets.
Package events defines the structures used for events relating directly to individual topology elements, like keyspaces, shards, and tablets.
Package helpers contains a few utility classes to handle topo.Server objects, and transitions from one topo implementation to another.
Package helpers contains a few utility classes to handle topo.Server objects, and transitions from one topo implementation to another.
TODO(sougou): The comments below look obsolete.
TODO(sougou): The comments below look obsolete.
faketopo
Package faketopo contains utitlities for tests that have to interact with a Vitess topology.
Package faketopo contains utitlities for tests that have to interact with a Vitess topology.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL