testing

Documentation

Index

• Variables
• type ChaosConfig
• type ChaosMonkey
  • func NewChaosMonkey(cluster *TestCluster, config ChaosConfig, logger forge.Logger) *ChaosMonkey
  • func (cm *ChaosMonkey) RunScenario(scenario ChaosScenario) error
  • func (cm *ChaosMonkey) Start(ctx context.Context)
  • func (cm *ChaosMonkey) Stop()
• type ChaosScenario
• type TestCluster
• type TestHarness
  • func NewTestHarness(t *testing.T) *TestHarness
  • func (h *TestHarness) AssertLeaderExists()
  • func (h *TestHarness) AssertNoLeader()
  • func (h *TestHarness) CreateCluster(ctx context.Context, numNodes int) error
  • func (h *TestHarness) CreatePartition(nodeIDs []string) error
  • func (h *TestHarness) GetHealthyNodeCount() int
  • func (h *TestHarness) GetLeader() (string, error)
  • func (h *TestHarness) GetNode(nodeID string) (*TestNode, error)
  • func (h *TestHarness) GetNodes() []*TestNode
  • func (h *TestHarness) GetQuorumSize() int
  • func (h *TestHarness) HealAllPartitions() error
  • func (h *TestHarness) HealPartition(nodeID string) error
  • func (h *TestHarness) InjectLatency(nodeID string, latency time.Duration) error
  • func (h *TestHarness) PartitionNode(nodeID string) error
  • func (h *TestHarness) RemoveLatency(nodeID string) error
  • func (h *TestHarness) StartCluster(ctx context.Context) error
  • func (h *TestHarness) StartNode(ctx context.Context, nodeID string) error
  • func (h *TestHarness) StopCluster(ctx context.Context) error
  • func (h *TestHarness) StopNode(ctx context.Context, nodeID string) error
  • func (h *TestHarness) SubmitToLeader(ctx context.Context, command []byte) error
  • func (h *TestHarness) WaitForCommit(index uint64, timeout time.Duration) error
  • func (h *TestHarness) WaitForLeader(timeout time.Duration) (string, error)
• type TestNode

Variables

var (
	ErrNoLeader = fmt.Errorf("no leader elected")
)

Predefined errors

var PredefinedScenarios = []ChaosScenario{
	{
		Name:        "LeaderFailure",
		Description: "Kill the leader node",
		Apply: func(cluster *TestCluster) error {
			leaderID, err := cluster.GetLeader()
			if err != nil {
				return nil
			}
			cluster.StopNode(context.Background(), leaderID)
			return nil
		},
		Verify: func(cluster *TestCluster) error {
			_, err := cluster.WaitForLeader(10 * time.Second)
			if err != nil {
				return ErrNoLeader
			}
			return nil
		},
		Duration: 10 * time.Second,
	},
	{
		Name:        "MinorityPartition",
		Description: "Partition minority of nodes",
		Apply: func(cluster *TestCluster) error {
			nodes := cluster.GetNodes()
			if len(nodes) < 3 {
				return nil
			}

			cluster.CreatePartition([]string{nodes[0].ID})
			return nil
		},
		Verify: func(cluster *TestCluster) error {

			_, err := cluster.WaitForLeader(10 * time.Second)
			if err != nil {
				return ErrNoLeader
			}
			return nil
		},
		Duration: 10 * time.Second,
	},
	{
		Name:        "MajorityPartition",
		Description: "Partition majority of nodes",
		Apply: func(cluster *TestCluster) error {
			nodes := cluster.GetNodes()
			if len(nodes) < 3 {
				return nil
			}

			partitionSize := (len(nodes) / 2) + 1
			partitionNodes := make([]string, partitionSize)
			for i := 0; i < partitionSize; i++ {
				partitionNodes[i] = nodes[i].ID
			}
			cluster.CreatePartition(partitionNodes)
			return nil
		},
		Verify: func(cluster *TestCluster) error {

			time.Sleep(5 * time.Second)
			return nil
		},
		Duration: 10 * time.Second,
	},
	{
		Name:        "CascadeFailure",
		Description: "Fail multiple nodes in sequence",
		Apply: func(cluster *TestCluster) error {
			nodes := cluster.GetNodes()
			if len(nodes) < 3 {
				return nil
			}

			for i := 0; i < len(nodes)-2; i++ {
				cluster.StopNode(context.Background(), nodes[i].ID)
				time.Sleep(2 * time.Second)
			}
			return nil
		},
		Verify: func(cluster *TestCluster) error {

			_, err := cluster.WaitForLeader(10 * time.Second)
			if err != nil {
				return ErrNoLeader
			}
			return nil
		},
		Duration: 15 * time.Second,
	},
	{
		Name:        "FlappingNode",
		Description: "Node repeatedly crashes and recovers",
		Apply: func(cluster *TestCluster) error {
			nodes := cluster.GetNodes()
			if len(nodes) == 0 {
				return nil
			}

			node := nodes[0]

			for i := 0; i < 5; i++ {
				cluster.StopNode(context.Background(), node.ID)
				time.Sleep(1 * time.Second)
				cluster.StartNode(context.Background(), node.ID)
				time.Sleep(1 * time.Second)
			}
			return nil
		},
		Verify: func(cluster *TestCluster) error {

			_, err := cluster.WaitForLeader(10 * time.Second)
			if err != nil {
				return ErrNoLeader
			}
			return nil
		},
		Duration: 15 * time.Second,
	},
}

PredefinedScenarios contains common chaos scenarios

Types

type ChaosConfig

type ChaosConfig struct {
	// Node failure probability (0.0 to 1.0)
	NodeFailureProbability float64
	// Network partition probability
	PartitionProbability float64
	// Latency injection probability
	LatencyProbability float64
	// Message drop probability
	MessageDropProbability float64
	// Check interval
	CheckInterval time.Duration
	// Min/Max latency to inject
	MinLatency time.Duration
	MaxLatency time.Duration
}

ChaosConfig contains chaos testing configuration

type ChaosMonkey

type ChaosMonkey struct {
	// contains filtered or unexported fields
}

ChaosMonkey implements chaos engineering for consensus testing

func NewChaosMonkey

func NewChaosMonkey(cluster *TestCluster, config ChaosConfig, logger forge.Logger) *ChaosMonkey

NewChaosMonkey creates a new chaos monkey

func (*ChaosMonkey) RunScenario

func (cm *ChaosMonkey) RunScenario(scenario ChaosScenario) error

RunScenario runs a chaos scenario

func (*ChaosMonkey) Start

func (cm *ChaosMonkey) Start(ctx context.Context)

Start starts the chaos monkey

func (*ChaosMonkey) Stop

func (cm *ChaosMonkey) Stop()

Stop stops the chaos monkey

type ChaosScenario

type ChaosScenario struct {
	Name        string
	Description string
	Apply       func(*TestCluster) error
	Verify      func(*TestCluster) error
	Duration    time.Duration
}

ChaosScenario represents a specific chaos scenario

type TestCluster

type TestCluster = TestHarness

TestCluster is an alias for TestHarness for backward compatibility

type TestHarness

type TestHarness struct {
	// contains filtered or unexported fields
}

TestHarness provides utilities for testing consensus

func NewTestHarness

func NewTestHarness(t *testing.T) *TestHarness

NewTestHarness creates a new test harness

func (*TestHarness) AssertLeaderExists

func (h *TestHarness) AssertLeaderExists()

AssertLeaderExists asserts that a leader exists

func (*TestHarness) AssertNoLeader

func (h *TestHarness) AssertNoLeader()

AssertNoLeader asserts that no leader exists

func (*TestHarness) CreateCluster

func (h *TestHarness) CreateCluster(ctx context.Context, numNodes int) error

CreateCluster creates a test cluster with the specified number of nodes

func (*TestHarness) CreatePartition

func (h *TestHarness) CreatePartition(nodeIDs []string) error

CreatePartition creates a network partition for the specified nodes

func (*TestHarness) GetHealthyNodeCount

func (h *TestHarness) GetHealthyNodeCount() int

GetHealthyNodeCount returns the number of started nodes

func (*TestHarness) GetLeader

func (h *TestHarness) GetLeader() (string, error)

GetLeader returns the current leader node ID

func (*TestHarness) GetNode

func (h *TestHarness) GetNode(nodeID string) (*TestNode, error)

GetNode returns a node by ID

func (*TestHarness) GetNodes

func (h *TestHarness) GetNodes() []*TestNode

GetNodes returns a slice of all test nodes

func (*TestHarness) GetQuorumSize

func (h *TestHarness) GetQuorumSize() int

GetQuorumSize returns the quorum size for the cluster

func (*TestHarness) HealAllPartitions

func (h *TestHarness) HealAllPartitions() error

HealAllPartitions heals all network partitions

func (*TestHarness) HealPartition

func (h *TestHarness) HealPartition(nodeID string) error

HealPartition reconnects a partitioned node

func (*TestHarness) InjectLatency

func (h *TestHarness) InjectLatency(nodeID string, latency time.Duration) error

InjectLatency injects artificial latency for a node

func (*TestHarness) PartitionNode

func (h *TestHarness) PartitionNode(nodeID string) error

PartitionNode simulates a network partition by disconnecting a node

func (*TestHarness) RemoveLatency

func (h *TestHarness) RemoveLatency(nodeID string) error

RemoveLatency removes artificial latency for a node

func (*TestHarness) StartCluster

func (h *TestHarness) StartCluster(ctx context.Context) error

StartCluster starts all nodes in the cluster

func (*TestHarness) StartNode

func (h *TestHarness) StartNode(ctx context.Context, nodeID string) error

StartNode starts a specific node

func (*TestHarness) StopCluster

func (h *TestHarness) StopCluster(ctx context.Context) error

StopCluster stops all nodes in the cluster

func (*TestHarness) StopNode

func (h *TestHarness) StopNode(ctx context.Context, nodeID string) error

StopNode stops a specific node (for partition testing)

func (*TestHarness) SubmitToLeader

func (h *TestHarness) SubmitToLeader(ctx context.Context, command []byte) error

SubmitToLeader submits a command to the leader

func (*TestHarness) WaitForCommit

func (h *TestHarness) WaitForCommit(index uint64, timeout time.Duration) error

WaitForCommit waits for a specific index to be committed on all nodes

func (*TestHarness) WaitForLeader

func (h *TestHarness) WaitForLeader(timeout time.Duration) (string, error)

WaitForLeader waits for a leader to be elected

type TestNode

type TestNode struct {
	ID           string
	RaftNode     internal.RaftNode
	Transport    internal.Transport
	Storage      internal.Storage
	StateMachine internal.StateMachine
	ClusterMgr   *cluster.Manager
	Discovery    internal.Discovery
	Started      bool
}

TestNode represents a node in the test cluster