benchmark

Documentation

Overview

Package benchmark owns Bomly's hidden local dependency-graph benchmark.

Index

• func FilterDocument(doc *sbom.Document, ecosystem sdk.Ecosystem) *sbom.Document
• func ParseNames(values ...string) []string
• func RenderText(w io.Writer, summary RunSummary) error
• type CaseSummary
• type NativeScanFunc
• type NativeScanRequest
• type NativeScanResult
• type PackageMetrics
• type PublicGitHubRepository
  • func ParsePublicGitHubRepository(value string) (PublicGitHubRepository, error)
• type RelationshipMetrics
• type RunOptions
• type RunSummary
  • func Run(ctx context.Context, opts RunOptions) (RunSummary, error)
• type ScopeSummary
• type ScoreSummary
• type SourceArtifacts
• type SourceSummary
  • func BuildSourceSummary(source string, bomlyDoc, sourceDoc *sbom.Document, artifacts SourceArtifacts) SourceSummary
• type Target
  • func LoadTargets(path string) ([]Target, error)
  • func Targets(targets []Target) []Target
  • func (t Target) SmokeArgs() []string

Functions

func FilterDocument

func FilterDocument(doc *sbom.Document, ecosystem sdk.Ecosystem) *sbom.Document

FilterDocument returns a copy containing only packages from ecosystem and their relationships.

func ParseNames

func ParseNames(values ...string) []string

ParseNames parses a comma-separated selector list.

func RenderText

func RenderText(w io.Writer, summary RunSummary) error

RenderText writes a compact benchmark score matrix.

Types

type CaseSummary

type CaseSummary struct {
	SchemaVersion string          `json:"schema_version"`
	Case          string          `json:"case"`
	Repository    string          `json:"repository"`
	HeadSHA       string          `json:"head_sha,omitempty"`
	Ecosystem     sdk.Ecosystem   `json:"ecosystem"`
	Status        string          `json:"status"`
	Reason        string          `json:"reason,omitempty"`
	Detectors     []string        `json:"used_detectors,omitempty"`
	Sources       []SourceSummary `json:"sources,omitempty"`
	Scores        *ScoreSummary   `json:"scores,omitempty"`
}

CaseSummary describes one repository comparison case.

type NativeScanFunc

type NativeScanFunc func(context.Context, NativeScanRequest) (NativeScanResult, error)

NativeScanFunc executes Bomly's native detectors without managed plugins or configuration files.

type NativeScanRequest

type NativeScanRequest struct {
	CheckoutDir  string
	Repository   string
	Revision     string
	Ecosystem    sdk.Ecosystem
	InstallFirst bool
}

NativeScanRequest describes one in-process Bomly native-detector scan.

type NativeScanResult

type NativeScanResult struct {
	Graph     *sdk.Graph
	Detectors []string
}

NativeScanResult contains the graph and detector provenance from one native scan.

type PackageMetrics

type PackageMetrics struct {
	BomlyCount      int     `json:"bomly_count"`
	SourceCount     int     `json:"source_count"`
	ExactMatches    int     `json:"exact_matches"`
	VersionMismatch int     `json:"version_mismatches"`
	BomlyOnly       int     `json:"bomly_only"`
	SourceOnly      int     `json:"source_only"`
	BomlyIgnored    int     `json:"bomly_ignored_without_purl"`
	SourceIgnored   int     `json:"source_ignored_without_purl"`
	Score           float64 `json:"score"`
}

PackageMetrics describes PURL-normalized package overlap.

type PublicGitHubRepository

type PublicGitHubRepository struct {
	URL  string
	Slug string
}

PublicGitHubRepository is a validated public GitHub repository selector.

func ParsePublicGitHubRepository

func ParsePublicGitHubRepository(value string) (PublicGitHubRepository, error)

ParsePublicGitHubRepository validates a public-repository URL shape.

type RelationshipMetrics

type RelationshipMetrics struct {
	BomlyCount  int      `json:"bomly_count"`
	SourceCount int      `json:"source_count"`
	Matched     int      `json:"matched"`
	BomlyOnly   int      `json:"bomly_only"`
	SourceOnly  int      `json:"source_only"`
	Score       *float64 `json:"score,omitempty"`
}

RelationshipMetrics describes PURL-normalized dependency-edge overlap.

type RunOptions

type RunOptions struct {
	ManifestPath       string
	RunDir             string
	SelectedCases      []string
	SelectedSources    []string
	SelectedEcosystems []string
	CustomRepository   string
	InstallFirst       bool
	Notifications      io.Writer
	HTTPClient         *http.Client
	Logger             *zap.Logger
	NativeScan         NativeScanFunc
}

RunOptions configures one hidden benchmark invocation.

type RunSummary

type RunSummary struct {
	SchemaVersion string        `json:"schema_version"`
	Status        string        `json:"status"`
	Reason        string        `json:"reason,omitempty"`
	RunDir        string        `json:"run_dir"`
	Cases         []CaseSummary `json:"cases,omitempty"`
	Scores        *ScoreSummary `json:"scores,omitempty"`
}

RunSummary describes a complete hidden benchmark invocation.

func Run

func Run(ctx context.Context, opts RunOptions) (RunSummary, error)

Run executes the hidden local benchmark and writes deterministic artifacts.

type ScopeSummary

type ScopeSummary struct {
	KnownScopeCount   int            `json:"known_scope_count"`
	UnknownScopeCount int            `json:"unknown_scope_count"`
	Scopes            map[string]int `json:"scopes,omitempty"`
}

ScopeSummary describes scope metadata availability for one SBOM source.

type ScoreSummary

type ScoreSummary struct {
	Package      float64  `json:"package"`
	Relationship *float64 `json:"relationship,omitempty"`
	Overall      float64  `json:"overall"`
}

ScoreSummary contains the benchmark scores for one comparison or aggregate.

type SourceArtifacts

type SourceArtifacts struct {
	SBOM     string `json:"sbom,omitempty"`
	RawSBOM  string `json:"raw_sbom,omitempty"`
	Diff     string `json:"diff,omitempty"`
	Log      string `json:"log,omitempty"`
	Response string `json:"response,omitempty"`
	Summary  string `json:"summary,omitempty"`
}

SourceArtifacts records paths relative to one benchmark case directory.

type SourceSummary

type SourceSummary struct {
	Source        string               `json:"source"`
	Status        string               `json:"status"`
	Reason        string               `json:"reason,omitempty"`
	Artifacts     SourceArtifacts      `json:"artifacts,omitempty"`
	Detectors     []string             `json:"used_detectors,omitempty"`
	Packages      *PackageMetrics      `json:"packages,omitempty"`
	Relationships *RelationshipMetrics `json:"relationships,omitempty"`
	BomlyScope    *ScopeSummary        `json:"bomly_scope,omitempty"`
	SourceScope   *ScopeSummary        `json:"source_scope,omitempty"`
	Scores        *ScoreSummary        `json:"scores,omitempty"`
}

SourceSummary describes one baseline comparison.

func BuildSourceSummary

func BuildSourceSummary(source string, bomlyDoc, sourceDoc *sbom.Document, artifacts SourceArtifacts) SourceSummary

BuildSourceSummary compares two filtered SBOM documents.

type Target

type Target struct {
	Name             string        `json:"name"`
	URL              string        `json:"url"`
	Ref              string        `json:"ref"`
	Ecosystem        sdk.Ecosystem `json:"ecosystem"`
	Args             []string      `json:"args,omitempty"`
	Tools            []string      `json:"tools,omitempty"`
	BenchmarkEnabled bool          `json:"benchmark_enabled,omitempty"`
}

Target describes one repository-backed smoke and benchmark case.

func LoadTargets

func LoadTargets(path string) ([]Target, error)

LoadTargets reads targets from path, or from the embedded manifest when path is empty.

func Targets

func Targets(targets []Target) []Target

Targets filters targets to those enabled for the hidden benchmark.

func (Target) SmokeArgs

func (t Target) SmokeArgs() []string

SmokeArgs returns the scan arguments for the target's pinned smoke-test revision.