shard

Documentation

Index

• func HashBytes(data []byte) uint64
• func HashString(key string) uint64
• type RendezvousHasher
  • func NewRendezvousHasher(shardCount int) *RendezvousHasher
  • func (rh *RendezvousHasher) GetShard(key uint64) int
• type Shard
• type ShardManager
  • func NewShardManager[T any](count int, shardFactory func() Shard[T]) *ShardManager[T]
  • func (sm *ShardManager[T]) Get(key uint64) (T, bool)
  • func (sm *ShardManager[T]) GetByString(key string) (T, bool)
  • func (sm *ShardManager[T]) GetShard(idx int) Shard[T]
  • func (sm *ShardManager[T]) Put(key uint64, item T) bool
  • func (sm *ShardManager[T]) PutByString(key string, item T) bool
  • func (sm *ShardManager[T]) ShardCount() int
  • func (sm *ShardManager[T]) TotalAvailable() int
  • func (sm *ShardManager[T]) TotalCapacity() int
• type SimpleHasher
  • func NewSimpleHasher(shardCount int) *SimpleHasher
  • func (sh *SimpleHasher) GetShard(key uint64) int

Functions

func HashBytes

func HashBytes(data []byte) uint64

purpose: converts bytes into a number for shard selection params: data []byte - the bytes we want to hash args: uses fnv64a hashing on the raw bytes returns: hash uint64 - the resulting hash number raises: none

func HashString

func HashString(key string) uint64

purpose: converts a string into a number we can use for picking shards params: key string - the text we want to convert args: uses a fast hashing algorithm (fnv64a) to turn the string into a number returns: hash uint64 - a big number representing the string raises: none

Types

type RendezvousHasher

type RendezvousHasher struct {
	// contains filtered or unexported fields
}

purpose: implements rendezvous hashing (highest random weight) to pick a shard - this gives

better distribution than simple modulo and handles shard count changes gracefully

params: key uint64 - the key to hash

shardCount int - how many shards we have

args: for each shard, calculates a score by combining the key and shard number, picks highest returns: shardIdx int - which shard won (had highest score) raises: none

func NewRendezvousHasher

func NewRendezvousHasher(shardCount int) *RendezvousHasher

purpose: creates a new rendezvous hasher for a given number of shards params: shardCount int - total number of shards args: stores the shard count for later use returns: pointer to RendezvousHasher - ready to hash keys raises: none

func (*RendezvousHasher) GetShard

func (rh *RendezvousHasher) GetShard(key uint64) int

purpose: picks the best shard for a key using highest-random-weight algorithm params: key uint64 - the key to place args: computes a score for each shard and returns the one with highest score returns: shardIdx int - the winning shard index raises: none

type Shard

type Shard[T any] interface {
	Get() (T, bool)
	Put(T) bool
	Available() int
	Capacity() int
}

purpose: represents one mini-pool in our collection of shards params: none (this is a type definition) args: n/a returns: n/a raises: n/a

type ShardManager

type ShardManager[T any] struct {
	// contains filtered or unexported fields
}

purpose: manages multiple mini-pools (shards) to reduce fighting between goroutines trying

to access the same pool - like having multiple checkout lines at a store

params: none (this is a type definition) args: n/a returns: n/a raises: n/a

func NewShardManager

func NewShardManager[T any](count int, shardFactory func() Shard[T]) *ShardManager[T]

purpose: creates a new shard manager with a specific number of shards params: count int - how many shards to create

shardFactory func() Shard[T] - a function that creates each individual shard

args: uses the factory to create each shard and sets up hashing to pick which shard to use returns: pointer to ShardManager - a manager controlling all the shards raises: none

func (*ShardManager[T]) Get

func (sm *ShardManager[T]) Get(key uint64) (T, bool)

purpose: gets an item from one of our shards using a key to pick which shard params: key uint64 - a number to help us choose which shard to use args: uses the key to pick a shard, then asks that shard for an item returns: (item T, ok bool) - the item we got and whether we succeeded raises: none

func (*ShardManager[T]) GetByString

func (sm *ShardManager[T]) GetByString(key string) (T, bool)

purpose: picks a shard based on a string instead of a number params: key string - a text key to choose a shard args: converts the string to a number using hashing, then picks the shard returns: (item T, ok bool) - the item we got raises: none

func (*ShardManager[T]) GetShard

func (sm *ShardManager[T]) GetShard(idx int) Shard[T]

purpose: gives direct access to a specific shard by its number params: idx int - which shard we want (0 to count-1) args: returns the shard at that position returns: shard Shard[T] - the shard at that index raises: none (will panic if idx is out of range)

func (*ShardManager[T]) Put

func (sm *ShardManager[T]) Put(key uint64, item T) bool

purpose: puts an item back into one of our shards params: key uint64 - a number to pick which shard

item T - the thing we're returning

args: uses the key to pick a shard, then gives the item to that shard returns: success bool - true if the item was stored raises: none

func (*ShardManager[T]) PutByString

func (sm *ShardManager[T]) PutByString(key string, item T) bool

purpose: returns an item to a shard chosen by a string key params: key string - text to identify which shard

item T - what we're returning

args: converts string to number, picks shard, stores item returns: success bool - true if stored raises: none

func (*ShardManager[T]) ShardCount

func (sm *ShardManager[T]) ShardCount() int

purpose: tells us how many shards we have params: none args: returns the count we set when creating the manager returns: count int - number of shards raises: none

func (*ShardManager[T]) TotalAvailable

func (sm *ShardManager[T]) TotalAvailable() int

purpose: counts the total number of available items across all shards params: none args: asks each shard how many items it has and adds them up returns: count int - total items available raises: none

func (*ShardManager[T]) TotalCapacity

func (sm *ShardManager[T]) TotalCapacity() int

purpose: counts the total capacity across all shards params: none args: asks each shard its capacity and sums them returns: cap int - total capacity of all shards combined raises: none

type SimpleHasher

type SimpleHasher struct {
	// contains filtered or unexported fields
}

purpose: a simple consistent hasher that just uses modulo - fast but less sophisticated params: none (this is a type definition) args: n/a returns: n/a raises: n/a

func NewSimpleHasher

func NewSimpleHasher(shardCount int) *SimpleHasher

purpose: creates a simple modulo-based hasher params: shardCount int - number of shards args: stores the count returns: pointer to SimpleHasher raises: none

func (*SimpleHasher) GetShard

func (sh *SimpleHasher) GetShard(key uint64) int

purpose: picks a shard using simple modulo division params: key uint64 - the key to place args: divides key by shard count and uses remainder returns: shardIdx int - which shard to use raises: none