mutation

package
Version: v0.0.0-...-0d16973 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 10, 2021 License: MIT Imports: 13 Imported by: 0

README

Mutation testing support

This package contains rudimentary support for mutation-testing compilers. At the moment, it's highly specialised to one particular test campaign using a run-time mutated LLVM, but any diversification is helpful.

Documentation

Overview

Package mutation contains support for mutation testing using c4t.

Index

Examples

Constants

View Source
const (
	// MutantHitPrefix is the prefix of lines from compilers specifying that a mutant has been hit.
	MutantHitPrefix = "MUTATION HIT:"
	// MutantSelectPrefix is the prefix of lines from compilers specifying that a mutant has been selected.
	MutantSelectPrefix = "MUTATION SELECTED:"
)
View Source
const EnvVar = "C4_MUTATION"

EnvVar is the environment variable used for mutation testing.

Some day, this might not be hard-coded.

Variables

View Source
var ErrNotActive = errors.New("automation is disabled in this config")

Functions

func ScanLine

func ScanLine(line string, onHit, onSelect func(Index))

ScanLine scans line for mutant hit and selection hints, and calls the appropriate callback.

func ScanLines

func ScanLines(r io.Reader) map[Index]uint64

ScanLines scans each line in r, building a map of mutant indices to hit counts. If a mutant is present in the map, it was selected, even if its hit count is 0.

Example

ExampleScanLines is a testable example for ScanLines.

package main

import (
	"fmt"
	"strings"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	lines := []string{
		"warning: overfull hbox",
		"MUTATION SELECTED: 42",
		"warning: ineffective assign",
		"MUTATION HIT: 42 (barely)",
		"info: don't do this",
		"this statement is false",
		"MUTATION SELECTED: 8",
		"MUTATION HIT: 42 (somewhat)",
	}

	for mutant, hits := range mutation.ScanLines(strings.NewReader(strings.Join(lines, "\n"))) {
		fmt.Println(mutant, "=", hits)
	}

}
Output:

42 = 2
8 = 0

Types

type Analysis

type Analysis map[Index]MutantAnalysis

Analysis is the type of mutation testing analyses.

func (Analysis) AddCompilation

func (a Analysis) AddCompilation(comp compilation.Name, log string, status status.Status)

AddCompilation merges any mutant information extracted from log to this analysis. Such analysis is filed under compilation name comp, and status determines the status of the compilation.

Example

ExampleAnalysis_AddCompilation is a testable example for AddCompilation.

package main

import (
	"fmt"
	"strings"

	"github.com/c4-project/c4t/internal/mutation"

	"github.com/c4-project/c4t/internal/subject/status"

	"github.com/c4-project/c4t/internal/id"
	"github.com/c4-project/c4t/internal/subject/compilation"
)

func main() {
	log := strings.Join([]string{
		"warning: overfull hbox",
		"MUTATION SELECTED: 42",
		"warning: ineffective assign",
		"MUTATION HIT: 42 (barely)",
		"info: don't do this",
		"this statement is false",
		"MUTATION SELECTED: 8",
		"MUTATION HIT: 42 (somewhat)",
	}, "\n")

	ana := mutation.Analysis{}
	ana.RegisterMutant(mutation.NamedMutant(42, "XYZ", 0))

	fmt.Println("kills after 0 adds:", ana.Kills())
	ana.AddCompilation(compilation.Name{SubjectName: "foo", CompilerID: id.FromString("gcc")}, log, status.Ok)
	fmt.Println("kills after 1 adds:", ana.Kills())
	ana.AddCompilation(compilation.Name{SubjectName: "bar", CompilerID: id.FromString("clang")}, log, status.Flagged)
	fmt.Println("kills after 2 adds:", ana.Kills())

	for _, ma := range ana {
		fmt.Printf("%s:", ma.Mutant)
		for _, h := range ma.Selections {
			fmt.Printf(" [%dx, %s, killed: %v]", h.NumHits, h.HitBy, h.Killed())
		}
		fmt.Println()
	}

}
Output:

kills after 0 adds: []
kills after 1 adds: []
kills after 2 adds: [XYZ:42]
XYZ:42: [2x, foo@gcc, killed: false] [2x, bar@clang, killed: true]
8: [0x, foo@gcc, killed: false] [0x, bar@clang, killed: false]

func (Analysis) Kills

func (a Analysis) Kills() []Mutant

Kills determines the mutants that were killed.

func (Analysis) RegisterMutant

func (a Analysis) RegisterMutant(m Mutant)

RegisterMutant registers the mutant record m in the analysis.

This is necessary, at the moment, to put things like the mutant's operator and variant information in the analysis table.

type AutoConfig

type AutoConfig struct {
	// Ranges contains the list of mutation number ranges that the campaign should use for automatic mutant selection.
	Ranges []Range `json:"ranges,omitempty" toml:"ranges,omitempty"`

	// ChangeMutantAfter is the (minimum) duration that each mutant gets before being automatically incremented.
	// If 0, this sort of auto-increment
	ChangeAfter quantity.Timeout `json:"change_after,omitempty" toml:"change_after,omitempty"`

	// ChangeKilled specifies whether mutants should be automatically incremented after being killed.
	ChangeKilled bool `json:"change_killed" toml:"change_killed"`
}

AutoConfig specifies configuration pertaining to automatically selecting mutants.

The mutation tester can be used with a manual selection, but is probably not very exciting.

func (AutoConfig) HasChangeAfter

func (c AutoConfig) HasChangeAfter() bool

HasChangeAfter gets whether ChangeAfter is set to something other than zero.

func (AutoConfig) HasRanges

func (c AutoConfig) HasRanges() bool

HasRanges gets whether at least one viable range exists, without expanding the ranges themselves.

func (AutoConfig) IsActive

func (c AutoConfig) IsActive() bool

IsActive gets whether automatic selection is enabled.

Example

ExampleAutoConfig_IsActive is a runnable example for Config.IsActive.

package main

import (
	"fmt"
	"time"

	"github.com/c4-project/c4t/internal/quantity"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	cfg := mutation.AutoConfig{
		Ranges: []mutation.Range{{Start: 1, End: 4}},
	}

	fmt.Println("disabled with ranges:", cfg.IsActive())

	cfg.ChangeKilled = true
	fmt.Println("after-killed with ranges:", cfg.IsActive())

	cfg.ChangeKilled = false
	cfg.ChangeAfter = quantity.Timeout(1 * time.Minute)
	fmt.Println("after-time with ranges:", cfg.IsActive())

	cfg.Ranges[0].Start = 4
	fmt.Println("after-time with bad ranges:", cfg.IsActive())

}
Output:

disabled with ranges: false
after-killed with ranges: true
after-time with ranges: true
after-time with bad ranges: false

func (AutoConfig) Mutants

func (c AutoConfig) Mutants() []Mutant

Mutants returns a list of all mutant numbers to consider in this testing campaign.

Mutants appear in the order defined, without deduplication. If Enabled is false, Mutants will be empty.

Example

ExampleAutoConfig_Mutants is a runnable example for Config.Mutants.

package main

import (
	"fmt"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	cfg := mutation.AutoConfig{
		Ranges: []mutation.Range{
			{Start: 1, End: 2},
			{Operator: "FOO", Start: 2, End: 3},
			{Start: 10, End: 12},
			{Operator: "BAR", Start: 27, End: 31},
		},
	}

	fmt.Print("mutants:")
	for _, i := range cfg.Mutants() {
		fmt.Printf(" %s", i)
	}
	fmt.Println()

}
Output:

mutants: 1 FOO:2 10 11 BAR1:27 BAR2:28 BAR3:29 BAR4:30

type AutoPool

type AutoPool struct {
	// contains filtered or unexported fields
}

AutoPool manages the next mutant to select in an automated mutation testing campaign.

The policy AutoPool implements is: 1) Start by considering every mutant in turn. 2) Whenever a mutant is killed or its timeslot ends, advance to the next mutant. 3) If we are out of mutants, start again with the list of all mutants not killed by the steps above, and repeat. 4) If we kill every mutant, start again with every mutant. (This behaviour may change eventually.)

func (*AutoPool) Advance

func (a *AutoPool) Advance()

Advance advances to the next mutant without killing it.

func (*AutoPool) Init

func (a *AutoPool) Init(muts []Mutant)

Init initialises this pool with initial mutants muts.

func (*AutoPool) Kill

func (a *AutoPool) Kill(m Mutant)

Kill marks the mutant m as killed, if it is the current mutant.

func (*AutoPool) Mutant

func (a *AutoPool) Mutant() Mutant

Mutant gets the currently selected mutant.

type Automator

type Automator struct {
	// TickerF is a stubbable function used to create a ticker.
	TickerF func(time.Duration) (<-chan time.Time, Ticker)
	// contains filtered or unexported fields
}

Automator handles most of the legwork of automating mutant selection.

func NewAutomator

func NewAutomator(cfg AutoConfig) (*Automator, error)

NewAutomator constructs a new Automator given configuration cfg.

func (*Automator) KillCh

func (a *Automator) KillCh() chan<- Mutant

KillCh gets a send channel for sending kill signals to this automator. If the automator isn't receiving kill signals, this will be nil. This channel must be closed.

func (*Automator) MutantCh

func (a *Automator) MutantCh() <-chan Mutant

MutantCh gets a receive channel for taking mutants from this automator.

func (*Automator) Run

func (a *Automator) Run(ctx context.Context)

Run runs this automator until ctx closes.

type Config

type Config struct {
	// Enabled gets whether mutation testing is enabled.
	//
	// Setting this to false is equivalent to setting Ranges to empty.
	Enabled bool `json:"enabled,omitempty" toml:"enabled,omitempty"`

	// Selection contains any selected mutation.
	//
	// This can be set in the tester's config file, in addition to or instead of Ranges, but will be overridden by
	// any automatic mutant selection.
	Selection Mutant `json:"selection,omitempty" toml:"selection,omitempty"`

	// Auto gathers configuration about how to automate mutation selection.
	Auto AutoConfig `json:"auto,omitempty" toml:"auto,omitempty"`
}

Config configures a particular mutation testing campaign.

This currently just tracks ranges of mutation numbers, but may be generalised if we branch to supporting more than one kind of mutation test.

type Index

type Index uint64

Index is the type of mutant indices.

type Mutant

type Mutant struct {
	// Name is the descriptive name of the mutant.
	Name Name

	// Index is the mutant index.
	//
	// The mutant index is what is passed into the mutation environment
	// variable, and is the basis for mutant definition by range.
	Index Index
}

Mutant is an identifier for a particular mutant.

Since we only support a mutation testing setups with integer mutant identifiers, this is just uint64 for now.

func AnonMutant

func AnonMutant(i Index) Mutant

AnonMutant creates a mutant with index i, but no name.

func NamedMutant

func NamedMutant(i Index, operator string, variant uint64) Mutant

NamedMutant creates a named mutant with index i, operator operator and variant variant. If operator is empty, the variant will not be recorded.

func (*Mutant) SetIndexIfZero

func (m *Mutant) SetIndexIfZero(i Index)

SetIndexIfZero sets this mutant's index to i if it is currently zero.

func (Mutant) String

func (m Mutant) String() string

String gets a human-readable string representation of this mutant.

If a name is available, the string will contain it.

Example

ExampleMutant_String is a runnable example for Mutant.String.

package main

import (
	"fmt"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	fmt.Println(mutation.NamedMutant(42, "", 0))
	fmt.Println(mutation.NamedMutant(56, "", 10))
	fmt.Println(mutation.NamedMutant(12, "FOO", 0))
	fmt.Println(mutation.NamedMutant(13, "BAR", 1))

}
Output:

42
56
FOO:12
BAR1:13

type MutantAnalysis

type MutantAnalysis struct {
	// Mutant contains the full record for this mutant.
	Mutant Mutant
	// Killed records whether this mutant was killed.
	Killed bool
	// Selections contains the per-selection analysis for this mutant.
	Selections []SelectionAnalysis
}

MutantAnalysis is the type of individual mutant analyses.

func (*MutantAnalysis) AddSelection

func (a *MutantAnalysis) AddSelection(sel SelectionAnalysis)

AddSelection adds sel to a's selection analyses.

type Name

type Name struct {
	// Operator is the name of the mutant operator, if given.
	Operator string
	// Variant is the index of this particular mutant within its operator.
	Variant uint64
}

Name is a human-readable name for mutants.

func (Name) IsZero

func (n Name) IsZero() bool

IsZero gets whether this name appears to be the zero value.

func (*Name) Set

func (n *Name) Set(operator string, variant uint64)

Set sets this name according to operator and variant. If operator is empty, we assume the mutant is unnamed, and clear the name to zero.

func (Name) String

func (n Name) String() string

String gets a string representation of this mutant name.

The zero name returns the empty string; otherwise, the name is the operator name followed, if nonzero, by the variant number.

type Range

type Range struct {
	// Operator is, if given, the name of the operator in this range.
	Operator string `json:"operator" toml:"operator"`

	// Start is the first mutant number to consider in this range.
	Start Index `json:"start" toml:"start"`
	// End is one past the last mutant number to consider in this range.
	End Index `json:"end" toml:"end"`
}

Range defines an inclusive numeric range of mutant numbers to consider.

func (Range) IsEmpty

func (r Range) IsEmpty() bool

IsEmpty gets whether this range defines no mutant numbers.

Example

ExampleRange_IsEmpty is a runnable example for IsEmpty.

package main

import (
	"fmt"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	fmt.Println("10..20:", mutation.Range{Start: 10, End: 20}.IsEmpty())
	fmt.Println("10..10:", mutation.Range{Start: 10, End: 10}.IsEmpty())
	fmt.Println("20..10:", mutation.Range{Start: 20, End: 10}.IsEmpty())
	fmt.Println("10..11:", mutation.Range{Start: 10, End: 11}.IsEmpty())

}
Output:

10..20: false
10..10: true
20..10: true
10..11: false

func (Range) IsSingleton

func (r Range) IsSingleton() bool

IsSingleton gets whether this range has exactly one item in it.

func (Range) Mutants

func (r Range) Mutants() []Mutant

Mutants expands a range into the slice of mutant numbers falling within it.

Example

ExampleRange_Mutants is a runnable example for Range.

package main

import (
	"fmt"

	"github.com/c4-project/c4t/internal/mutation"
)

func main() {
	fmt.Print("unnamed:")
	for _, i := range (mutation.Range{Start: 10, End: 20}).Mutants() {
		fmt.Printf(" %s", i)
	}
	fmt.Println()
	fmt.Print("named:  ")
	for _, i := range (mutation.Range{Operator: "ABC", Start: 20, End: 23}).Mutants() {
		fmt.Printf(" %s", i)
	}
	fmt.Println()

}
Output:

unnamed: 10 11 12 13 14 15 16 17 18 19
named:   ABC1:20 ABC2:21 ABC3:22

type SelectionAnalysis

type SelectionAnalysis struct {
	// Timespan represents the timespan at which the compilation finished.
	Timespan timing.Span `json:"time_span"`

	// NumHits is the number of times this compilation hit the mutant.
	// If this is 0, the mutant was selected but never hit.
	NumHits uint64 `json:"num_hits"`

	// Status was the main status of the compilation, which determines whether the selection killed the mutant.
	Status status.Status `json:"status"`

	// HitBy is the name of the compilation that hit this mutant.
	HitBy compilation.Name `json:"by"`
}

SelectionAnalysis represents one instance where a compilation selected a particular mutant.

func (SelectionAnalysis) Hit

func (h SelectionAnalysis) Hit() bool

Hit gets whether this selection resulted in at least one hit.

func (SelectionAnalysis) Killed

func (h SelectionAnalysis) Killed() bool

Killed gets whether this selection resulted in a kill (hit at least once and resulted in a flagged status).

type Ticker

type Ticker interface {
	Reset(d time.Duration)
	Stop()
}

Ticker is a mockable interface for time.Ticker.

func StandardTicker

func StandardTicker(d time.Duration) (<-chan time.Time, Ticker)

StandardTicker gets a standard Go ticker if d is nonzero, and a no-op otherwise. In the latter case, the returned channel is nil.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL