Documentation
¶
Overview ¶
Package kernel is the AgentOS v2 kernel core: the typed object model, the object-store ABI, the append-only journal, the measured resource model, admission control, the single scheduler, and lease primitives.
This package is the shared ABI for the rest of the kernel (controllers, durable lanes, substrate, the gRPC API). Phases 2-6 build against the public types and interfaces declared here, so they are a contract: keep them clean.
Phase 1 ships an in-memory object store and journal only; the etcd backend is a later phase. Standard library only.
The object model is split across focused files (the ≤500-line legibility invariant): this file holds the shared envelope (ObjectMeta, Status, Condition, the Object interface, the deep-copy seam); object_tickets.go, object_runtime.go, object_leases.go, object_resource.go, and object_apps.go hold the concrete kinds. No exported identifier moved — only file location.
Index ¶
- Constants
- Variables
- func AdvanceLifecycleRedriveLedger(labels map[string]string, now time.Time) int
- func ApplyRunningLaneAccounting(ledger *ResourceLedger, running []RunningLane)
- func BoundedStderrTail(s string) string
- func ClassifyOutcome(exitCode int) string
- func ClearExclusiveContention(labels map[string]string)
- func ClearLaneActivityBlockedCondition(status *Status)
- func ClearLaneRetryCondition(status *Status)
- func ClearLifecycleRedriveLedger(labels map[string]string)
- func DefaultSchedulerScopeLeaseTTL(leaseTTL time.Duration) time.Duration
- func DeleteCAS(store Store, obj Object) error
- func EffectiveTarget(admitted []Admission) int
- func FilesOutsideWriteScope(scopes []WriteScope, files []string) []string
- func GetMany(store Store, kind Kind, names []string) (map[string]Object, error)
- func IsExclusiveCurrency(c Currency) bool
- func IsFiniteNonNegAmount(x float64) bool
- func IsIndexedLabelKey(key string) bool
- func IsIntegralCurrency(c Currency) bool
- func IsKnownKind(kind Kind) bool
- func IsWholeNonNegToken(x float64) bool
- func LaneActivityBlockedBackoffActive(lane *Lane, now time.Time) (time.Time, bool)
- func LaneHasValidPublishedArtifact(lane *Lane) bool
- func LanePublishedArtifactRef(lane *Lane) string
- func LaneRetryBackoffActive(lane *Lane, now time.Time) (time.Time, bool)
- func LaneTicketPhaseValue(ticket, phase string) string
- func LifecycleRedriveCount(labels map[string]string, now time.Time) int
- func PathScopeCovers(scope WriteScope, file string) bool
- func ProcessHeartbeat(p Process) time.Time
- func Reconcile(runnable []Ticket, ledger *ResourceLedger) (admitted []Admission, waiting []Ticket)
- func RegisterExclusiveCurrency(c Currency)
- func RegisterExecutionResourceCurrency(c Currency)
- func RegisterExtensionKind(kind Kind, ctor func() Object) error
- func RegisterIndexedLabelKey(key string) error
- func RegisterIntegralCurrency(c Currency)
- func SchedulerLeaseService(cell string) string
- func SetLaneRetryCondition(status *Status, state DurableRetryState, exhausted bool)
- func SetProcessHeartbeat(p *Process, ts time.Time)
- func SetProcessMeasuredUsage(p *Process, used ResourceRequest)
- func StampExclusiveContention(labels map[string]string, currencies []Currency, at time.Time)
- func StampLanePublishedArtifact(lane *Lane, artifactType, artifactRef string, at time.Time)
- func TerminalFailureSignature(exitCode int, reason string) string
- func TerminalLaneOwnedResourceTxnPlan(store Store, lane *Lane) (LaneOwnedResourceReleaseResult, []Cond, []Op, error)
- func TerminalLaneOwnedResourceTxnPlanWithOptions(store Store, lane *Lane, opts ...TerminalLaneOwnedResourceTxnPlanOption) (LaneOwnedResourceReleaseResult, []Cond, []Op, error)
- func TerminateRequested(st Status) bool
- func TicketNameFromLaneName(laneName string) string
- func TicketPlacementDoneCell(status string) (string, bool)
- func TicketSchedulingOwnedStatus(status string) bool
- func ValidateEpoch(currentEpoch, writeEpoch int64) error
- func ValidateTxnPutFencing(conds []Cond, ops []Op) error
- func ValidateTxnScopeOpSet(ops []Op) error
- func WriteScopesConflict(a, b WriteScope) bool
- type ActivitySummary
- type Admission
- type AdmissionGate
- type App
- type AppInstall
- type AppInstallSpec
- type AppSpec
- type AppendStats
- type BatchedGetter
- type Cell
- type CellDirectory
- type CellDirectoryEntry
- type CellDirectoryPublishOptions
- type CellDirectorySpec
- type CellPlacementResult
- type CellPlacer
- type CellPlacerConfig
- type CellSpec
- type CellStoreResolver
- type Clock
- type CloudProvider
- type Cond
- type CondType
- type Condition
- type ConditionStatus
- type ConflictError
- type ConsumerFailureRecord
- type Currency
- type Decision
- type DurableRetryState
- type Event
- type EventType
- type ExternalResourceReleaseFunc
- type FenceRef
- type FencedWriter
- type ForcedDropRecord
- type Journal
- type JournalRetentionStats
- type JournalRetentionStatsReader
- type Kind
- type Lane
- type LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceExternalRelease(fn ExternalResourceReleaseFunc) LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceFence(fence FenceRef) LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceSkipExternalRelease() LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceSkipProcessDiscovery() LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceTrimProviderToken() LaneOwnedResourceReleaseOption
- func WithLaneOwnedResourceWriter(writer LaneResourceWriter) LaneOwnedResourceReleaseOption
- type LaneOwnedResourceReleaseResult
- func ReleaseLaneExecutionResources(store Store, lane *Lane, proc *Process, opts ...LaneOwnedResourceReleaseOption) (LaneOwnedResourceReleaseResult, error)
- func ReleaseLaneOwnedResources(store Store, lane *Lane, proc *Process, opts ...LaneOwnedResourceReleaseOption) (LaneOwnedResourceReleaseResult, error)
- type LaneResourceWriter
- type LaneSpec
- type LaneState
- type LeaseFencing
- type LeaseManager
- func (lm *LeaseManager) AcquireScope(ws WriteScope, holder string, ttl time.Duration) (*WriteScopeLease, error)
- func (lm *LeaseManager) AcquireScopeEpoch(ws WriteScope, holder string, ttl time.Duration, epoch int64) (*WriteScopeLease, error)
- func (lm *LeaseManager) AcquireScopeFenced(ws WriteScope, holder string, ttl time.Duration, epoch int64, cell string, ...) (*WriteScopeLease, error)
- func (lm *LeaseManager) AcquireScopeInCell(ws WriteScope, holder string, ttl time.Duration, epoch int64, cell string) (*WriteScopeLease, error)
- func (lm *LeaseManager) AcquireScopeLeaseByIdentity(id WriteScopeLeaseIdentity, ttl time.Duration, epoch int64, fence FenceRef) (*WriteScopeLease, error)
- func (lm *LeaseManager) AcquireService(service, holder string, ttl time.Duration) (*ServiceLease, error)
- func (lm *LeaseManager) AcquireServiceInstance(service, holder, instanceID string, ttl time.Duration) (*ServiceLease, error)
- func (lm *LeaseManager) ReleaseScopeInstance(ws WriteScope, holder, instanceID string) error
- func (lm *LeaseManager) ReleaseScopeInstanceInCell(ws WriteScope, holder, cell, instanceID string) error
- func (lm *LeaseManager) ReleaseScopeLeaseByIdentity(id WriteScopeLeaseIdentity, writer LaneResourceWriter) (bool, error)
- func (lm *LeaseManager) ReleaseService(service, holder string) error
- func (lm *LeaseManager) ReleaseServiceInstance(service, instanceID string) error
- func (lm *LeaseManager) RenewScopeLeaseByIdentity(id WriteScopeLeaseIdentity, ttl time.Duration, epoch int64, fence FenceRef) (*WriteScopeLease, error)
- func (lm *LeaseManager) RenewService(service, holder string) (*ServiceLease, error)
- func (lm *LeaseManager) RenewServiceInstance(service, instanceID string) (*ServiceLease, error)
- func (lm *LeaseManager) ScopeLeaseByIdentity(id WriteScopeLeaseIdentity) (*WriteScopeLease, error)
- func (lm *LeaseManager) ServiceLeaseEpoch(service string) (epoch int64, live bool, err error)
- func (lm *LeaseManager) ServiceLeaseFencing(service string) (LeaseFencing, error)
- type ListOptions
- type ListPage
- type MemJournal
- func (j *MemJournal) Append(ev Event) (int64, error)
- func (j *MemJournal) AppendStats() AppendStats
- func (j *MemJournal) BeginGroupApply(group string) (func(), error)
- func (j *MemJournal) ClearConsumerFailure(group string, sourceOffset int64) error
- func (j *MemJournal) Commit(group string, cursor int64) error
- func (j *MemJournal) CommittedCursor(group string) (int64, error)
- func (j *MemJournal) EnforceRetention(forceDropHorizon int64) (RetentionResult, error)
- func (j *MemJournal) Head() (int64, error)
- func (j *MemJournal) Lag(group string) (int64, error)
- func (j *MemJournal) Poll(group string) ([]Event, error)
- func (j *MemJournal) PollN(group string, limit int) ([]Event, error)
- func (j *MemJournal) RecordConsumerFailure(group string, source Event, maxAttempts int, cause string) (ConsumerFailureRecord, error)
- func (j *MemJournal) RetentionStats() (JournalRetentionStats, error)
- func (j *MemJournal) RetireDeadGroups() []string
- func (j *MemJournal) Subscribe(group string) error
- func (j *MemJournal) Unsubscribe(group string) error
- type MemStore
- func (s *MemStore) AcquireScopeLease(lease *WriteScopeLease, cell string, fence *FenceRef) (Object, error)
- func (s *MemStore) Create(obj Object) (Object, error)
- func (s *MemStore) Delete(kind Kind, name string) error
- func (s *MemStore) DeleteVersioned(kind Kind, name string, expectVersion int64) error
- func (s *MemStore) FencedCreate(obj Object, fence FenceRef) (Object, error)
- func (s *MemStore) FencedUpdate(obj Object, fence FenceRef) (Object, error)
- func (s *MemStore) Get(kind Kind, name string) (Object, error)
- func (s *MemStore) List(kind Kind, sel Selector) ([]Object, error)
- func (s *MemStore) ListPage(kind Kind, sel Selector, opts ListOptions) (ListPage, error)
- func (s *MemStore) LiveScopeLeasesByHolder(cell, holder string) ([]*WriteScopeLease, error)
- func (s *MemStore) ReleaseScopeLease(leaseName string, scope WriteScope, holder, cell, instanceID string) error
- func (s *MemStore) RenewScopeLeaseIdentity(id WriteScopeLeaseIdentity, ttl time.Duration, epoch int64, fence *FenceRef) (Object, error)
- func (s *MemStore) SetClock(clock Clock)
- func (s *MemStore) Stats() StoreStats
- func (s *MemStore) Txn(conds []Cond, ops []Op) error
- func (s *MemStore) Update(obj Object) (Object, error)
- func (s *MemStore) Watch(kind Kind, stop <-chan struct{}) (<-chan WatchEvent, error)
- func (s *MemStore) WatchFrom(kind Kind, snapshotRev int64, stop <-chan struct{}) (<-chan WatchEvent, error)
- func (s *MemStore) WatchSnapshot(kind Kind, sel Selector, limit int, stop <-chan struct{}) (ListPage, <-chan WatchEvent, error)
- type MetricGauge
- type Node
- type NodeLifecycle
- type NodeSpec
- type NodeStatus
- type Object
- type ObjectMeta
- type OfferSource
- type Op
- type OpType
- type OwnerRef
- type PhaseStat
- type PoisonLaneReason
- type Preemption
- type Preemptor
- type Process
- type ProcessSpec
- type ProcessStatusExt
- type ProgressGauge
- type ReconcileInput
- type RefillSource
- type ResourceGrant
- type ResourceGrantSpec
- type ResourceLedger
- func (l *ResourceLedger) ApplyReclamation(overuse ResourceRequest)
- func (l *ResourceLedger) ApplyReservation(amounts ResourceRequest)
- func (l *ResourceLedger) Available(c Currency) float64
- func (l *ResourceLedger) CanReserve(req ResourceRequest) bool
- func (l *ResourceLedger) Capacity(c Currency) float64
- func (l *ResourceLedger) Clone() *ResourceLedger
- func (l *ResourceLedger) RechargeUsage(delta ResourceRequest)
- func (l *ResourceLedger) Release(req ResourceRequest)
- func (l *ResourceLedger) Reservable(c Currency) float64
- func (l *ResourceLedger) Reserve(req ResourceRequest) error
- func (l *ResourceLedger) Reserved(c Currency) float64
- func (l *ResourceLedger) Shortfalls(req ResourceRequest) []Currency
- func (l *ResourceLedger) Snapshot() map[Currency]ResourceLine
- func (l *ResourceLedger) Used(c Currency) float64
- type ResourceLine
- type ResourceOffer
- type ResourceRequest
- type RetentionEnforcer
- type RetentionResult
- type RetryBudgetPolicy
- type ReversiblePreemptor
- type RevisionWatcher
- type RunnableSource
- type RunningLane
- type RunningSource
- type Scheduler
- type SchedulerConfig
- type ScopeConflictGuard
- type ScopeKind
- type Selector
- type ServiceLease
- type ServiceLeaseSpec
- type SnapshotWatcher
- type Spawner
- type SpawnerStore
- type Status
- type Store
- type StoreLabelNormalizer
- type StoreStats
- type TerminalEvidence
- type TerminalLaneOwnedResourceTxnPlanOption
- type TickLaneSnapshot
- type TickLaneSnapshotConsumer
- type TickPhase
- type TickPhaseStats
- type TickResult
- type Ticket
- type TicketSpec
- type Tier
- type ToolCount
- type ValidatableObject
- type Verdict
- type WaitCondition
- type WaitConditionKind
- type WaitHandle
- type WaitNotifier
- type WaitRegistry
- type WaitResult
- type WaitStats
- type WatchEvent
- type WatchEventType
- type WriteScope
- type WriteScopeLease
- type WriteScopeLeaseIdentity
- type WriteScopeLeaseSpec
Constants ¶
const ( // ExclusiveContentionLatchLabel marks a ticket carrying a live exclusive- // contention stamp, with the fixed value "true" so the scheduler can find // every stamped ticket with one exact-match indexed label list (the same // bounded-list shape the orphan reconciler uses for its blocker classes) — // never a full ticket scan. ExclusiveContentionLatchLabel = "agentos.dev/exclusive-contended" // ExclusiveContendedCurrenciesLabel records WHICH exclusive currencies the // stamped ticket's request named, comma-joined, sorted. The latch derivation // reads it back; admission gates exactly these currencies. ExclusiveContendedCurrenciesLabel = "agentos.dev/exclusive-contended-currencies" // ExclusiveContendedAtLabel is the RFC3339Nano observation time of the // backpressure exit the stamp records. A stamp older than the contention // TTL is STALE and derives no latch (the stale-proof adversarial case): it // is evidence about a hold that may have ended, so it must not gate forever. ExclusiveContendedAtLabel = "agentos.dev/exclusive-contended-at" // WaitingExclusiveContended is the typed WaitingOn reason for a ticket held // at admission because an exclusive currency it requests is contended by an // out-of-band holder (fresh backpressure evidence). Distinct from the bare // currency name (a ledger shortfall): the currency LOOKS free in the ledger, // and the kernel is refusing to spend a lane attempt proving otherwise. WaitingExclusiveContended = "exclusive_currency_contended" // DefaultExclusiveContentionTTL bounds how long backpressure evidence latches // admission before the kernel spends ONE probe attempt to re-measure. It is // also therefore the worst-case admission delay after an out-of-band holder // releases. Configurable per scheduler via SchedulerConfig. // Round-2: deliberately LONGER than DefaultExclusiveIdleHoleThreshold // (10m) — the contention latch must outlive the idle-hole watchdog's // observation window, or a latch expiring exactly at the watchdog // threshold lets one probe admission race the severity-high idle-hole // emission (emission-vs-probe ordering). DefaultExclusiveContentionTTL = 15 * time.Minute )
const ( // LaneArtifactPublishedConditionType is the durable lane Condition a worker // stamps when its artifact is fully produced AND every required-evidence file // is present — i.e. the deliverable EXISTS and only verification/closeout // remains. Its Reason carries a short artifact-type hint (e.g. "git_commit", // "benchmark_report"); its Message carries the artifact reference. Recovery // reads it to decide re-verify-vs-recompute. LaneArtifactPublishedConditionType = "ArtifactPublished" // LaneArtifactPublishedLabel mirrors the ArtifactPublished condition onto an // indexed label (value LaneArtifactPublishedValue) so a controller can find // "lanes that published an artifact" with one indexed read rather than a // status scan — the same producer contract as LanePhaseLabel. LaneArtifactPublishedLabel = "agentos.dev/artifact-published" // LaneArtifactPublishedValue is the LaneArtifactPublishedLabel value for a // lane whose artifact + required evidence are present. LaneArtifactPublishedValue = "true" // LaneArtifactRefLabel records the published artifact's reference (the // content-addressed id / branch / report path the verifier re-rules). Present // iff LaneArtifactPublishedLabel is set; recovery surfaces it so the re-verify // path names the exact artifact it is re-ruling rather than recomputing. LaneArtifactRefLabel = "agentos.dev/artifact-ref" )
const ( // CondSignalReceived is the Condition.Type the signal API stamps on a // process/lane target when an operator signal is recorded. CondSignalReceived = "SignalReceived" // SignalReasonTerminate is the Condition.Reason carried by an operator // terminate signal. The string matches the public signal kind "terminate". SignalReasonTerminate = "terminate" // LaneTerminatedReason is the terminal Phase-condition reason a lane carries // after an operator terminate was honoured. It is deliberately distinct from // every genuine-failure reason so recovery policy can tell "operator stopped // this" apart from "the workload failed". LaneTerminatedReason = "operator_terminate" )
lane_terminate.go — the durable OPERATOR-TERMINATE mark and its single reader-side definition.
The bug it closes (live incident 2026-06-09): `agentosctl signal -target lane -kind terminate` stamped a SignalReceived condition on the Lane and appended a journal event — and NOTHING consumed either. The lane's provider process kept running; when the operator killed it by hand the lane workflow treated the death as an ordinary failed attempt and respawned a fresh process (the structural MaxLaneAttempts loop), so the only terminal outcome reachable was attempt-cap ESCALATION counted as a genuine workload failure.
This file gives the mark ONE definition both sides share:
- PRODUCER: the signal API (pkg/api, pkg/grpcapi) stamps a CondSignalReceived condition with Reason SignalReasonTerminate on the Lane's Status. The condition is durable, CAS-written, and /proc-visible.
- CONSUMERS: the lane execution path gates every agent (re)spawn on TerminateRequested and escalates a TERM→KILL onto the live process; the orphan reconciler routes a terminated lane's ticket to an operator-owned block instead of the respawn ledger. Producer and consumers cannot disagree about what "terminate requested" means because this is the only definition.
The mark is a Status condition (not a label) deliberately: it is stamped by the existing signal write path, survives every lane phase transition (phase writers only upsert their own condition types), and is read at attempt boundaries by name — never by indexed list — so no new index is needed.
const ( // ActivitySummaryStatusLive — the attempt is still running and observed. ActivitySummaryStatusLive = "live" // ActivitySummaryStatusAttaching — the adapter is attached but no record has // arrived yet (e.g. waiting for the worker's session banner). Counters in // this state are NOT measurements; /proc renders the status, not zeros. ActivitySummaryStatusAttaching = "attaching" // ActivitySummaryStatusTerminal — the attempt ended and the aggregate below // is the final measured truth. ActivitySummaryStatusTerminal = "terminal" // observed; UnavailableReason names why. Counters MAY hold partial data // observed before the adapter died (Records > 0) — never fabricated zeros. ActivitySummaryStatusUnavailable = "unavailable" )
const ( // TerminalEvidenceStderrTailCap bounds the captured stderr tail to ~2 KB so a // runaway worker log can never bloat a durable object. The runtime truncates // to the LAST whole lines that fit (a tail, not a head) — the failure cause // is at the end of the log. TerminalEvidenceStderrTailCap = 2048 // TerminalEvidenceStderrTailLines bounds the tail to the last N lines as well, // so even a single enormous line cannot exceed the intent of "a short tail". TerminalEvidenceStderrTailLines = 40 )
const ( // OutcomeCompleted — the workload finished its work (exit 0). OutcomeCompleted = "completed" // OutcomeFailed — a genuine crash/error. The respawn backoff/cap owns it. OutcomeFailed = "failed" // OutcomeBackpressure — the workload could NOT run because a resource it // needs is busy/held (e.g. an exclusive GPU lease). NOT a failure: the work // is sound, it just has to wait. Recovery REQUEUES it without counting toward // the respawn fail-cap, so a singleton-exclusive resource never churns // "failed" lanes that were merely queued behind the holder. OutcomeBackpressure = "backpressure" // OutcomeKilled — terminated by a signal (exit > 128). External kill / OOM. OutcomeKilled = "killed" // OutcomeTerminated — stopped by an OPERATOR terminate signal (the durable // TerminateRequested mark). NOT a workload failure: recovery must neither // count it toward the respawn fail-cap nor auto-redrive the ticket — the // operator explicitly stopped this work, and only an explicit operator // redrive/retire moves it again. Distinct from OutcomeKilled, which is an // UNATTRIBUTED signal death (an OOM, a stray kill) and stays on the failure // recovery path. OutcomeTerminated = "terminated" // OutcomeUnknown — unclassifiable; recovery treats it as OutcomeFailed. OutcomeUnknown = "unknown" )
Terminal OutcomeClass values classify WHY a lane ended so recovery is class-appropriate. The set is deliberately small + domain-agnostic.
const ( // PlacementMarkerLabel is the Ticket label mirroring placement-marker // statuses (placed-to-*, copied-to-*, done-in-*). CellPlacer recovery // selects this exact indexed label instead of prefix-scanning the Ticket // kind for marker status strings. PlacementMarkerLabel = "agentos.dev/placement-marker" // PlacementMarkerValue is the exact indexed value used for placement // markers. PlacementMarkerValue = "true" // LanePhaseLabel is the Lane label mirroring Status.Phase (PENDING, // RUNNING, VERIFYING, DONE, ...). The lane state machine writes it on every // phase transition; controllers select on it. LanePhaseLabel = "agentos.dev/lane-phase" // LaneMergeReadyLabel is the Lane label mirroring the MergeReady condition. // Its presence with value LaneMergeReadyValue means the verifier verdict // passed and the branch is ready to land — the merger selects DONE + this. LaneMergeReadyLabel = "agentos.dev/merge-ready" // LaneMergeReadyValue is the LaneMergeReadyLabel value for a merge-ready // lane. LaneMergeReadyValue = "true" // LaneMergeReadyConditionType is the durable release-gate condition that // says a terminal artifact is ready for the merger/release controller. LaneMergeReadyConditionType = "MergeReady" // LaneMergedConditionType is the durable post-state proof that the release // controller landed the artifact. LaneMergedConditionType = "Merged" // LaneMergeRejectedConditionType is the durable terminal proof that the // release controller rejected the artifact. LaneMergeRejectedConditionType = "MergeRejected" // LaneTicketLabel is the Lane label mirroring LaneSpec.TicketName — the // name of the Ticket this lane is the durable execution of. A Lane's own // Name is lane-<ticket>-<seq>, so a controller that wants an INDEXED read // of "the lane(s) for ticket X" (the pm controller resolves a dependency // edge by the upstream ticket's lane state) must select on this label // rather than Get a lane by ticket name. The spawner writes it on lane // creation; LaneSpec.TicketName stays the authoritative field a consumer // re-checks, exactly as LanePhaseLabel mirrors Status.Phase. LaneTicketLabel = "ticket" // LaneTicketPhaseLabel is the composite distributed absence-fence index for // "ticket X has a lane in phase Y". It lets controllers atomically prove no // live lane for a ticket exists without scanning all lanes or blocking on // unrelated tickets in the same phase. LaneTicketPhaseLabel = "agentos.dev/lane-ticket-phase" )
Lane indexed-label contract (§7.2 hot-path rule). The kernel store indexes ObjectMeta.Labels, not Status — a label Selector cannot see Status.Phase or Status.Conditions. A controller that wants an INDEXED read of "every lane in phase X" (the verifier wants VERIFYING lanes, the merger wants DONE + merge-ready lanes) must select on a label, so whatever writes Status.Phase MUST mirror it onto LanePhaseLabel in the same write. These constants are the one kernel-owned definition of that contract: the workflow lane state machine (the producer) and the verifier/merger controllers (the consumers) both reference them, so the producer and the consumer cannot disagree about the label key.
const ( // LaneRetryWaitingOn is the visible WaitingOn reason for a RUNNING lane // parked behind durable retry backoff. LaneRetryWaitingOn = "activity_retryable" // LaneRetryEscalatedWaitingOn is the visible WaitingOn reason after the // durable retry budget is exhausted. LaneRetryEscalatedWaitingOn = "activity_retry_budget_exhausted" // LaneRetryConditionType carries durable retry state in Condition.Message. LaneRetryConditionType = "ActivityRetryable" // LaneActivityBlockedWaitingOn parks a lane behind external proof or // ownership resolution. It is not a retryable failure and must not consume // retry budget. LaneActivityBlockedWaitingOn = "activity_blocked" // LaneActivityBlockedConditionType records that an activity reached an // external proof/ownership wait. LaneActivityBlockedConditionType = "ActivityBlocked" // LaneRetryReasonTransient means the retry budget still has room. LaneRetryReasonTransient = "TransientActivityFailure" // LaneRetryReasonBudgetExhausted means the retry budget was exhausted. LaneRetryReasonBudgetExhausted = "ActivityRetryBudgetExhausted" // DefaultActivityBlockedBackoff is the durable scheduler hold after an // activity-blocked checkpoint. The hold prevents every-tick re-submission // while still periodically re-driving work whose blocker cleared. DefaultActivityBlockedBackoff = 30 * time.Second )
const ( // AgentStopLabel is the indexed Ticket label that durably stops a single // agent (its ticket/lane). When present with AgentStopValue the scheduler // refuses to admit the ticket and surfaces it WAITING on agentStopWaitReason. // It is an additive key on the ticket's open label set (the frozen-ABI rule // permits adding to an open set), in the same `agentos.dev/*` namespace as the // admission marker and redrive counter. The value is a fixed boolean rather // than per-instance so any controller can test "is this agent stopped?" with a // single indexed equality, never a prefix scan. AgentStopLabel = "agentos.dev/agent-stop" // AgentStopValue is the only value AgentStopLabel carries when active. Clearing // the stop DELETES the label (resume) rather than writing a second value, so // the runnable/admission predicate is a simple key-presence + value check. AgentStopValue = "true" // AgentStopReasonLabel optionally records a short human/audit reason for the // stop, mirroring PauseReason on the org-pause. It is advisory only — the // scheduler gates purely on AgentStopLabel — so a missing reason never changes // admission behaviour. AgentStopReasonLabel = "agentos.dev/agent-stop-reason" )
const ( // LifecycleRedriveCountLabel is the windowed lifecycle-loss count. LifecycleRedriveCountLabel = "agentos.dev/lifecycle-redrive-count" // LifecycleRedriveAtLabel is the RFC3339 stamp of the LAST lifecycle loss; // the count above is valid only within LifecycleRedriveWindow of it. LifecycleRedriveAtLabel = "agentos.dev/lifecycle-redrive-at" // MaxLifecycleRedrives bounds lifecycle re-queues per window. MaxLifecycleRedrives = 5 // LifecycleRedriveWindow is the sliding window the count lives in. LifecycleRedriveWindow = time.Hour )
Lifecycle-loss redrive ledger — the WINDOWED budget for re-queuing a ticket whose scheduler binding was lost with NO genuine failure (a lost process or grant, a pending-start timeout, an interrupted lane, a no-lane orphan).
Why windowed and not lifetime: lifecycle losses are routine across daemon restarts — a long-lived ticket legitimately accumulates them forever, so a LIFETIME cap turns ordinary operations into a permanent tombstone (the 2026-06 wedge: every `blocked/lane_terminal_no_active_workload` ticket was born with its budget already spent, operator redrive refused, only etcd label surgery exited). What the budget must actually catch is a LOOP — the pending-timeout shape that closes a lane and re-queues the same ticket every ~2 minutes, invisible to every cap because each cycle looks like a fresh admission. A loop is "many losses in a short window"; the ledger therefore counts losses within LifecycleRedriveWindow and DECAYS once the window slides past the last loss.
The decay doubles as the recovery mechanic: a ticket blocked at the cap reads count==0 one window later, so the orphan reconciler's next pass re-queues it without any separate recovery-window machinery. Worst case a permanently-looping ticket gets cap attempts per window (bounded, visible, breadcrumbed) instead of one per ~2 minutes forever; a ticket that stopped looping recovers by itself. Genuine FAILURES are deliberately not this ledger — they key the consecutive same-reason respawn cap (pkg/control/controllers), whose post-cap rescues draw a CUMULATIVE budget.
Both the kernel's own recovery family (resetTicketAndCloseOldLaneWithReason) and the orphan reconciler's lifecycle redrive path write THIS ledger, so a loop cannot hide by alternating between the two actors (the S1-5 bypass: scheduler-side recoveries previously incremented nothing at all).
const ( TriageLabel = "status" TriageValue = "triage" // AdmittedValue marks a ticket the queen has committed this tick. A ticket // with this status is no longer runnable; its durable lane drives it from // here. The lane state machine owns any later status transition. AdmittedValue = "admitted" // BlockedValue marks a ticket whose upstream dependency edges are not yet // resolved. The scheduler's indexed RunnableSource selects ONLY TriageValue, // so a ticket carrying BlockedValue is visibly queued-but-not-runnable — // the §14b "no hidden blockers" rule. The pm controller writes this value // (its pmBlockedValue is this constant) when it gates a ticket out of the // runnable queue, and a dependency-bearing ticket is BORN with this value // so the scheduler cannot admit it before the pm controller's first // reconcile tick gates it: a ticket submitted WITH a non-empty DependsOn // must never be momentarily admissible. It is an additive value on the open // `status` label set (the frozen-ABI rule permits adding to an open set). BlockedValue = "blocked" // TicketCompletedValue is the terminal status of a ticket whose work // genuinely finished: latest lane DONE and (for git lanes) merged. Settled // by the orphan reconciler so completion memory lives on the TICKET and // survives lane-record GC. Deliberately NOT the cell-placement // `done-in-<cell>` family: that status is a PLACEMENT MARKER with its own // lifecycle (the placer normalizes it into a marker label, ages it, RESETS // stale ones, and eventually DELETES the ticket object) — reusing it for // local completion would hand settled tickets to the placer's marker GC. TicketCompletedValue = "completed" // RetiredValue marks work an operator intentionally removed from the live // queue. It is not runnable, and dependency resolution treats it as // unresolved so retiring an upstream ticket cannot accidentally release // downstream work. RetiredValue = "retired" )
TriageLabel / TriageValue is the Ticket label the scheduler's runnable read indexes on — the same label the pm controller's triage view selects, so the scheduler and the PM share one definition of "a queued ticket".
AdmittedValue is the TriageLabel value a ticket carries once the admission transaction has committed it (finding #3). The admission transaction CAS-transitions the admitted ticket from TriageValue to AdmittedValue inside the same transaction that issues its grants and spawns its lane, so the indexed runnable read no longer returns it — a later tick cannot re-select the same ticket, re-admit it, and spawn a SECOND lane against the same grant. It is an additive value on the open `status` label set (the frozen-ABI rule permits adding to an open set).
const ( // AdmissionActiveLabel is the exact indexed mirror for "ticket has any // AdmissionMarkerLabel". AdmissionMarkerLabel's value is unique per // scheduler instance/epoch, so controllers that need all marked tickets use // this boolean mirror instead of prefix-scanning the Ticket kind. AdmissionActiveLabel = "agentos.dev/admission-active" // AdmissionActiveValue is the exact value used for AdmissionActiveLabel. AdmissionActiveValue = "true" )
const ( // TierLabel is the Lane label mirroring the owning ticket tier for lane // budget accounting. TierLabel = "tier" // ClassLaneLabel is the explicit, scheduler-owned lane-budget class. Scope // and program remain legacy/client-owned fallbacks. ClassLaneLabel = "agentos.dev/lane-class" // ClassScopeLabel and ClassProgramLabel are the legacy bounded class labels // the lane-budget policy reads when assigning class capacity. ClassScopeLabel = "scope" ClassProgramLabel = "program" // WorkloadLabel is a client-defined workload-kind label used for routing and // policy selection. The kernel indexes the key but does not own values like // "training", "research_agent", or any product-specific workload taxonomy. WorkloadLabel = "workload" // ArtifactTypeLabel declares the verifier artifact contract for non-git // and generic workloads. Admission copies it from Ticket to Lane so terminal // claim completion can still resolve the contract if the Ticket disappears. ArtifactTypeLabel = "agentos.dev/artifact-type" // ProcessLaneLabel links a Process row to the Lane it runs. ProcessLaneLabel = "lane" // ProcessTicketLabel links a Process row to the owning Ticket. It is the // typed distributed-ownership index; controllers must prefer it over // reverse-parsing Lane names. ProcessTicketLabel = "agentos.dev/ticket" )
const ActivitySummaryMetricCap = 12
ActivitySummaryMetricCap bounds the per-summary named-gauge breakdown to N names so a pathological non-LLM worker emitting an unbounded set of metric names cannot bloat the durable Lane object. Metrics are gauges (not counts), so the bound keeps the first N by name (deterministic), not top-N by value.
const ActivitySummaryToolCallCap = 8
ActivitySummaryToolCallCap bounds the per-summary tool-name breakdown to the top-N names by count; the remainder is folded into the "_other" bucket.
const AdmissionMarkerLabel = "agentos.dev/admitted-by"
AdmissionMarkerLabel is the indexed ObjectMeta.Label the admission transaction stamps onto a ticket it CAS-transitions out of the runnable queue. Its value identifies the exact queen instance + leadership epoch that bound the ticket. A later admission that sees the marker treats the ticket as already claimed and fails cleanly rather than creating a duplicate lane. It is an additive label key (the frozen-ABI rule permits adding to an open set).
const CellLabel = "agentos.io/cell"
CellLabel is the indexed ObjectMeta.Label under which a Ticket records its assigned §15.4 cell and a Process/Lane records the cell it belongs to. A queen of one cell never claims another cell's tickets or folds another cell's running lanes into its ledger. Tickets with no CellLabel are default work; Process writes are normalized to an explicit DefaultCell label so the RunningSource hot path can stay indexed.
const CellPlacerService = "cell-placer"
CellPlacerService is the single-activation ServiceLease name for the global ticket placement authority. It migrates unowned/default-cell triage tickets from the global queue into named cells; it does not admit or schedule work.
const CompletedLaneLeaseReapedConditionType = "CompletedLaneLeaseReaped"
CompletedLaneLeaseReapedConditionType is the typed lane Condition the runtime stamps when it detects a COMPLETED-LANE LEASE HOLD — a lane whose activity rollup is still LIVE (the wrapper keeps heartbeating, the PID is alive) but whose latest phase carries a TERMINAL lifecycle status (the worker's own heartbeat declared the run done: phase.status=="terminal", e.g. a trainer's completed / EARLY_STOP heartbeat) AND which is still alive past the configured grace — and reaps it so the held resource (GPU/MPS) lease frees and the daemon can finalize → promote → mint the next stage. It is the third sibling of the reaper family: ZombieTerminalReapedConditionType closes "terminal RECEIPT file present + still heartbeating"; TrainingProgressStallConditionType closes "RUNNING + heartbeating + progress frozen + NO terminal evidence"; THIS one closes the gap between them — "the worker's HEARTBEAT says terminal-completed + still alive holding a lease + NO terminal receipt written yet" (live incident 2026-06-13, pid 33891: a trainer that COMPLETED its run hung in teardown for 12+ minutes at 0% CPU holding its MPS lease, before it ever wrote the process_terminal.v1 receipt the zombie watch keys on, while the progress-stall watch never fires on a terminal phase). Generic: the discriminator is the rollup's own live status + the phase's coarse terminal lifecycle marker; the kernel never names a workload, a phase vocabulary, or a metric.
const DefaultCell = "default"
DefaultCell is the cell id of the single-cell kernel-MVP. §15.4: a cell is the unit of consistency — one resource ledger, one Process authority, one scheduler (one queen). The MVP runs one cell; scale is more cells.
const DefaultExclusiveIdleHoleThreshold = 10 * time.Minute
DefaultExclusiveIdleHoleThreshold is the default persistence window before a free-with-waiting-demand exclusive currency emits the typed hole event.
const DefaultForceDropHorizonFactor = 4
DefaultForceDropHorizonFactor sets the default force-drop horizon as a multiple of maxRetained: a group is force-dropped only once its lag is this many times the steady-state retention cap, i.e. it is hopelessly behind. The factor keeps the horizon proportional to the configured retention so a large journal tolerates proportionally larger transient lag before backpressure.
const DefaultJournalRetention = 100_000
DefaultJournalRetention is the live-log cap if NewJournal is given 0.
const DefaultWaitTimeout = 30 * time.Second
DefaultWaitTimeout bounds a Wait that passes a non-positive timeout.
const ExitBackpressure = 87
ExitBackpressure is the AgentOS-standard exit code a workload uses to signal "I could not run because a resource I need is busy/held — requeue me; this is NOT a failure." (Canonical case: an exclusive GPU-training lease already held by a sibling lane.) The kernel maps it to OutcomeBackpressure so the lane is requeued without counting as a terminal failure. Any workload may adopt it; it describes the OUTCOME, not a domain. Chosen to match the LEASE_REFUSED exit code the training entrypoint already emits.
const ForcedDropReason = "force_drop_retention"
ForcedDropReason is the EventDeadLetter payload "reason" for a retention-driven forced cursor advance, distinct from a per-event poison dead-letter.
const HeartbeatConditionType = "Heartbeat"
HeartbeatConditionType is the kernel.Condition Type under which a Process's last heartbeat is recorded on the durable Process object.
HEARTBEAT-CADENCE CONTRACT (§7.7 store/log split, resolved). A process beat is a HIGH-CHURN record. The authoritative beat STREAM is the journal: the runtime publishes an EventHeartbeat per cadence and the reaper/`/proc` consume that stream. The durable Process object carries only an OCCASIONAL last-beat CHECKPOINT as a Condition of this type — written on Register and on coarse liveness-state transitions, NOT per fine-grained cadence. A writer MUST NOT CAS-write the Process object on every beat: that is the v1 churn that made etcd fail. ProcessHeartbeat/SetProcessHeartbeat are the one kernel-owned definition of where the checkpoint lives so the runtime writer, the reaper controller, and the `/proc` projector cannot drift; the per-beat path is Journal.Append(EventHeartbeat), never Store.Update(Process).
const MaxLaneAttempts = 3
MaxLaneAttempts is the structural rework cap (§7.6): a lane reaches DONE or ESCALATED within this many attempts, by construction.
const MeasuredUsageConditionType = "MeasuredUsage"
MeasuredUsageConditionType is the kernel.Condition Type under which a Process's last measured per-currency resource usage is checkpointed on the durable Process object. It is the Borg-reclamation (§7.3) input the scheduler reads when it rebuilds the ledger: a lane whose measured Used exceeds its granted reservation is re-charged at the overage so admission tightens against the real footprint.
HEARTBEAT-CADENCE CONTRACT (the same store/log split as HeartbeatConditionType). The high-churn measured-usage STREAM is the journal; the durable Process object carries only an OCCASIONAL last-measurement CHECKPOINT in a Condition of this type. A writer MUST NOT CAS-write the Process on every measurement sample — that is the v1 churn that broke etcd. The Condition's Message encodes the per-currency amounts as `currency=amount` pairs (see SetProcessMeasuredUsage); ProcessMeasuredUsage decodes them. The kernel-shipped indexedRunningSource reads this checkpoint so reclamation works against the durable store without a measurement field on ProcessSpec.
const PreemptionRestoreFailedConditionType = "PreemptionRestoreFailed"
PreemptionRestoreFailedConditionType is the kernel.Condition Type the admission rollback stamps onto a victim Process whose preemption could NOT be restored after the replacement admission failed (Round-7 finding #3). The verifier/reconciler — which reconciles durable object state — selects/scans for a Process carrying this condition and recovers the stranded victim. It makes a failed restoration a durable, visible fact rather than a discarded error. It is an additive condition type (the frozen-ABI rule permits adding to an open set).
const ReanchorDropReason = "cursor_reanchor_compaction"
ReanchorDropReason is the EventDeadLetter payload "reason" for a cursor that fell behind compaction (a TTL-dead group whose cursor stopped pinning, then revived via Subscribe/Poll/BeginGroupApply) and was re-anchored to the retention floor. The skipped range was never delivered to the group, so the re-anchor is a drop and MUST be recorded — a consumer-group cursor never passes an un-acked offset silently (H3: no silent audit loss).
const RetiredWithLagReason = "group_retired_with_lag"
RetiredWithLagReason is the EventDeadLetter payload "reason" recorded when RetireDeadGroups deletes a consumer group that still had un-acked events (cursor < head). The exact future loss is unknowable at retirement (it depends on later compaction), so the marker records the un-acked lag as the upper bound; a revival before compaction replays from the retained log, a revival after sees only this marker.
const RunningPhaseLabel = LanePhaseLabel
RunningPhaseLabel / runningPhaseValues are the Lane phase-label values the scheduler's running read indexes on — a lane that is consuming resources.
const SchedulerName = "scheduler"
SchedulerName is the scheduler service-lease name of the single-cell kernel-MVP — the DEFAULT cell's queen lease (§15.1). It is an exported constant of the frozen ABI; its value is unchanged.
const TrainingProgressStallConditionType = "TrainingProgressStall"
TrainingProgressStallConditionType is the typed lane Condition the runtime stamps when it detects a PROGRESS-STALL — a lane that is RUNNING with a fresh heartbeat (the activity rollup is live) yet whose reported progress gauge (the "how far along" Current the worker emits — e.g. a trainer's iter) has NOT advanced for longer than the configured grace, while NOT in a declared long-running phase (a corpus refresh / preflight legitimately reports no progress). It is the sibling of ZombieTerminalReapedConditionType: that one closes "terminal evidence present + still heartbeating"; this one closes "RUNNING + heartbeating + progress frozen + NO terminal evidence" — the FAKE-RUNNING training lane (live incident 2026-06-13: a trainer wedged post-preflight with a fresh heartbeat and a live PID, never advancing an iteration, so the lab never relaunched it). Generic: the contradiction is defined entirely by the kernel's own liveness + activity-progress contracts; the kernel never names a workload, a phase, or a metric.
const TrainingWarmupOverrunConditionType = "TrainingWarmupOverrun"
TrainingWarmupOverrunConditionType is the typed lane Condition the runtime stamps when a phase that is EXEMPT from the progress-stall reaper (a declared long-running phase the watch never reaps — warmup / preflight / corpus / model-load) has been resident for longer than the configured overrun grace WITHOUT advancing into a should-be-advancing phase and WITHOUT reporting any finer-grained sub-progress.
This condition is ADVISORY / NON-TERMINAL by deliberate contract: UNLIKE ZombieTerminalReapedConditionType, TrainingProgressStallConditionType, and CompletedLaneLeaseReapedConditionType — all of which accompany a reap — this condition NEVER reaps, signals, or terminates the process. The exempt phase continues to suppress the reaper exactly as before; the condition only makes a suspiciously-long warmup VISIBLE to an operator (live incident 2026-06-07: a 9B critic warmup that wedged at iter=0 for 20+ minutes was invisible because "warmup" is infinitely exempt). A legitimate long warmup (a 9B model load + resume-bundle restore) is expected to exceed an ordinary step gap, so the overrun grace is set well above it and a worker that reports warmup sub-progress re-seeds the timer and is never stamped. Generic: the kernel names no workload, phase vocabulary, or metric — only "an exempt phase has been resident, with no sub-progress, past the grace".
const ZombieTerminalReapedConditionType = "ZombieTerminalReaped"
ZombieTerminalReapedConditionType is the typed lane Condition the runtime stamps when it detects the zombie-terminal contradiction — a process that reported terminal evidence (a terminal=true process_terminal.v1 receipt) yet kept running (heartbeat fresh) past the configured grace — and reaps it, terminalizing the lane from the RECORDED terminal evidence rather than the kill's signal exit. (Live incident 2026-06-12: a worker hung post-completion for 4h19m holding an exclusive resource token while its wrapper kept heartbeating.) Generic: the contradiction is defined entirely by the kernel's own receipt + liveness contracts; no workload vocabulary.
Variables ¶
var ( // ErrUnknownGroup is returned when a consumer operation names a group // that never Subscribed. ErrUnknownGroup = errors.New("kernel: unknown consumer group") // ErrCursorRegressed is returned by Commit when the supplied cursor is // behind the group's already-committed cursor — commits only move // forward. ErrCursorRegressed = errors.New("kernel: commit cursor regressed") )
Journal-layer error sentinels.
var ( // ErrLeaseHeld is returned by an acquire attempt that lost the race — // another holder owns a live lease. ErrLeaseHeld = errors.New("kernel: lease already held") // ErrNotHolder is returned by Renew/Release when the caller is not the // current holder (or, for the instance-fenced variants, does not present // the current lease-instance token). ErrNotHolder = errors.New("kernel: caller is not the lease holder") // ErrNotLockable is returned when a WriteScope names a non-lockable // (prose) scope kind. Prose strings are never lockable (§6). ErrNotLockable = errors.New("kernel: write scope kind is not lockable") // ErrScopeConflict is returned when a write scope overlaps a live lease // held by another owner. ErrScopeConflict = errors.New("kernel: write scope conflicts with a held lease") // ErrStaleEpoch is returned when a leader-authored mutating write carries a // LeadershipEpoch behind the current queen's epoch — the §15.1 fencing // rejection. A stale former leader that resumed after its lease expired // hits this on every correctness-bearing write. ErrStaleEpoch = errors.New("kernel: stale leadership epoch — write fenced") // ErrLeadershipLost is returned by the §15.1 admission-transaction fence // when the queen no longer holds a LIVE lease at the queen's own epoch and // instance — the lease expired, was released, or a successor re-acquired // it. It is distinct from ErrStaleEpoch (which is only the behind-epoch // case): a successor that re-used the same epoch NUMBER, or a queen whose // lease simply lapsed, leaves the bare epoch matching while leadership is in // fact gone. ValidateLeadership rejects a write the instant that is true. ErrLeadershipLost = errors.New("kernel: leadership lost — write fenced") // ErrTicketNotRunnable is returned by the admission transaction's // markTicketAdmitted (Round-7 finding #2) when the ticket it is about to // admit is no longer a runnable durable object — it is absent from the store, // or no longer carries TriageLabel=TriageValue. Either means a concurrent // leader/admission already claimed it; the admission must ABORT (no preempt, // no spawn) rather than transition a precondition it never actually checked // and spawn a duplicate lane. ErrTicketNotRunnable = errors.New("kernel: ticket is not runnable — concurrent admission claimed it") // ErrWriteScopeBudget is returned by admission when a ticket's WriteScope set // would produce a Store.Txn larger than the backend transaction-op ceiling // (etcd --max-txn-ops). Refusing it deterministically — in MemStore and // EtcdStore alike — prevents the dual-mode divergence where local admission // succeeds but the same ticket fails permanently in etcd (Round-9 scale #4). ErrWriteScopeBudget = errors.New("kernel: ticket write-scope set exceeds the admission transaction op budget") // ErrPreemptorNotReversible is returned by the admission transaction // (Round-7 finding #3) when an admission needs to preempt a victim but the // configured Preemptor does not implement ReversiblePreemptor. Enacting an // irreversible preemption behind a spawn that can still fail would strand the // victim preempted with no admitted replacement; the transaction refuses to // preempt and the admission falls back to waiting. ErrPreemptorNotReversible = errors.New("kernel: preemption needed but the configured Preemptor is not reversible") )
Lease-layer error sentinels.
var ( // ErrNotFound is returned by Get/Update/Delete for an absent object. ErrNotFound = errors.New("kernel: object not found") // ErrAlreadyExists is returned by Create for a duplicate (Kind, Name). ErrAlreadyExists = errors.New("kernel: object already exists") // ErrConflict is returned by Update when the supplied ResourceVersion is // stale — a compare-and-swap loser. The caller re-reads and retries. ErrConflict = errors.New("kernel: resource version conflict") )
Store-layer error sentinels. Callers test with errors.Is.
var AllCurrencies = []Currency{ CurrencyWorkspaceSlot, CurrencyRAMBytes, CurrencyCPU, CurrencyProviderToken, CurrencyScopeLease, CurrencyGPUToken, CurrencyMPSExclusive, CurrencyMemHeavy, CurrencyStoreWriteOps, }
AllCurrencies is the canonical ordered list of currencies. Iterating it gives a deterministic BlockedOn order.
var ErrBadConstructor = fmt.Errorf("kernel: extension kind constructor produced an unusable object")
ErrBadConstructor is returned by RegisterExtensionKind when the supplied constructor produces an unusable Object — a nil interface, or an interface carrying a typed-nil pointer ((*T)(nil) boxed as Object). Either would later panic NewObject when it stamps Kind via GetMeta(); the registry rejects the constructor at registration time rather than admitting a latent kernel crash.
var ErrFenceStale = errors.New("kernel: fenced write rejected — fence object moved (stale leader)")
ErrFenceStale is returned by a fenced write whose fence object no longer matches the expected identity — the §15.1 atomic stale-leader rejection. It is distinct from ErrConflict (the writer's OWN object lost a CAS) and from ErrStaleEpoch / ErrLeadershipLost (the pre-write leadership re-read): ErrFenceStale means the fence and the write were evaluated together and the fence had moved. A caller treats it exactly like a leadership fence — the queen no longer leads, so the whole admission aborts.
var ErrFenceUnsupported = errors.New("kernel: store backend does not support fenced writes")
ErrFenceUnsupported is returned by FencedCreate / FencedUpdate when the Store backend does not implement FencedWriter. A fenced write must be ATOMIC with its fence; a backend that cannot guarantee that atomicity must fail the write CLOSED rather than silently fall back to an unfenced Create/Update — a silent fallback would reintroduce exactly the TOCTOU gap the primitive closes. Every production backend (MemStore, EtcdStore) implements it.
var ErrKindAlreadyRegistered = fmt.Errorf("kernel: object kind already registered")
ErrKindAlreadyRegistered is returned by RegisterExtensionKind when a Kind is already known — a kernel-owned kind, or an extension kind a different caller already registered. Kind collisions are a build-time bug, surfaced loudly.
var ErrKindTypeMismatch = errors.New("kernel: object Kind does not match its concrete type")
ErrKindTypeMismatch is returned by a store write when an object's ObjectMeta.Kind does not match its concrete Go type — e.g. a *Process whose Kind was set to "Ticket". Storing it would make a later type assertion (scheduler.go, lease.go do `obj.(*Ticket)`) panic at read time. The store enforces Kind↔concrete-type congruence at write time so a mismatched object never enters the store.
var ErrLaneExecutionReleaseNotTerminal = errors.New("kernel: lane execution release requires terminal lane")
ErrLaneExecutionReleaseNotTerminal means compute-only resource release was requested before the lane crossed a durable no-further-attempt boundary.
var ErrLaneResourceReleaseBlocked = errors.New("kernel: lane-owned resource release blocked")
ErrLaneResourceReleaseBlocked means lane cleanup proved a lane-owned resource is still unsafe to release, for example a provider claim whose subprocess has not reached a terminal result. Callers should leave the lane/process envelope intact and retry after the owning reconciler proves absence or completion.
var ErrTypedNilObject = errors.New("kernel: object interface carries a typed-nil pointer")
ErrTypedNilObject is returned by a store write handed an interface that is non-nil but carries a typed-nil pointer (e.g. (*Ticket)(nil) boxed into an Object). Such a value passes `obj == nil` yet panics on the first method call — the store rejects it at the gate.
var ErrUnknownKind = fmt.Errorf("kernel: unknown object kind")
ErrUnknownKind is returned by NewObject for a Kind with no registered constructor — neither a kernel-owned durable kind nor a registered extension kind.
var IndexedLabelKeys = map[string]struct{}{ CellLabel: {}, TriageLabel: {}, PlacementMarkerLabel: {}, AdmissionMarkerLabel: {}, AdmissionActiveLabel: {}, LanePhaseLabel: {}, LaneMergeReadyLabel: {}, LaneTicketLabel: {}, LaneTicketPhaseLabel: {}, TierLabel: {}, ClassLaneLabel: {}, ClassScopeLabel: {}, ClassProgramLabel: {}, ProcessLaneLabel: {}, ProcessTicketLabel: {}, ExclusiveContentionLatchLabel: {}, "agentos.dev/health-active": {}, "agentos.dev/controller": {}, }
IndexedLabelKeys is the bounded allowlist of ObjectMeta label keys that the store backends maintain as physical secondary indexes. It intentionally contains only scheduler/controller hot-path labels; arbitrary user labels remain correct via post-filtering but are not indexed.
Treat this map as read-only. It is exported so every backend and oracle test shares one source of truth for the index contract.
Functions ¶
func AdvanceLifecycleRedriveLedger ¶
AdvanceLifecycleRedriveLedger records one more lifecycle loss on the labels (restarting from 1 when the window has slid past the previous losses) and returns the new in-window count.
func ApplyRunningLaneAccounting ¶
func ApplyRunningLaneAccounting(ledger *ResourceLedger, running []RunningLane)
ApplyRunningLaneAccounting folds already-running lanes into ledger exactly as the scheduler does at the start of every tick. It is the shared resource accounting seam for any read surface that must project the scheduler's effective ledger: durable ResourceGrants become Reserved, and measured overuse becomes additional Reserved via Borg-style reclamation.
func BoundedStderrTail ¶
BoundedStderrTail returns the LAST whole lines of s that fit within both the byte cap and the line cap — a tail, never unbounded. The failure cause is at the end of a log, so a tail (not a head) is the useful diagnostic. The result is whole lines (it never splits mid-line at the byte cap) and carries no trailing newline.
func ClassifyOutcome ¶
ClassifyOutcome maps a process exit code to a typed OutcomeClass — the single place the exit-code → outcome convention lives so producer and consumer cannot disagree. A signal death (>128) is OutcomeKilled; ExitBackpressure is OutcomeBackpressure; any other non-zero is OutcomeFailed.
func ClearExclusiveContention ¶
ClearExclusiveContention removes the contention stamp — called when a ticket converges (its lane completed: the exclusive resource was demonstrably acquirable) so a settled ticket does not carry latch evidence forever. Expiry handles every other case: a stale stamp simply derives no latch.
func ClearLaneActivityBlockedCondition ¶
func ClearLaneActivityBlockedCondition(status *Status)
ClearLaneActivityBlockedCondition removes the external-proof wait marker without touching retry budget state.
func ClearLaneRetryCondition ¶
func ClearLaneRetryCondition(status *Status)
ClearLaneRetryCondition removes retryable-activity state from Status.
func ClearLifecycleRedriveLedger ¶
ClearLifecycleRedriveLedger removes the ledger — written on genuine success (ticket completion) and on an explicit operator redrive (a human decision is a clean slate).
func DefaultSchedulerScopeLeaseTTL ¶
DefaultSchedulerScopeLeaseTTL returns the kernel-level write-scope lease default for direct embedders. Service leadership failover stays short (LeaseTTL), while running-lane ownership gets a wider window for scheduler jitter and distributed-store latency.
func DeleteCAS ¶
DeleteCAS is a version-guarded delete over the Store ABI: it deletes obj only if the store still holds obj's ResourceVersion. It is the safe counterpart to the blind Store.Delete — a stale caller loses with a *ConflictError instead of erasing a newer object.
If the backend implements versionedDeleter (MemStore does), the delete is genuinely ATOMIC. Otherwise DeleteCAS falls back to Get-then-Delete: it re-reads the object, compares the version, and deletes — a narrow race remains for a backend without a native CAS-delete, which is documented and acceptable because the kernel's own backend (MemStore) takes the atomic path and a future etcd backend can implement versionedDeleter with an etcd txn.
func EffectiveTarget ¶
EffectiveTarget reports the emergent lane target for a tick: it is exactly the count of admitted tickets, never a configured number. It exists so /proc can surface "the kernel intends to run N lanes because N is what measured resources admit", honouring the effective-target-honesty SLO (§11).
func FilesOutsideWriteScope ¶
func FilesOutsideWriteScope(scopes []WriteScope, files []string) []string
FilesOutsideWriteScope returns, in input order, the files NOT covered by any path WriteScope in scopes. It is the deterministic core of the merge ship-gate's diff-in-scope guard: a lane whose committed diff touches a file outside its declared write scope is worker over-reach — e.g. a formatter rewriting an unrelated crate, or a stray deletion — and the merger rejects it terminally, REGARDLESS of what an LLM critic concluded.
When scopes declares NO path scope at all there is no path contract to enforce, so the result is empty and the caller imposes no scope constraint: the gate enforces a declared contract, it does not invent one. Path scopes are author-set on the ticket, not worker-controlled, so an unscoped lane is a policy gap to fix at ticket creation, not a worker evasion vector.
func GetMany ¶
GetMany reads many objects of one Kind from a Store. If the Store implements BatchedGetter it is used directly (one round-trip); otherwise a per-key Store.Get fallback preserves correctness against any backend.
Either path returns one map entry per name that exists. A missing name is omitted (it is NOT treated as an error). A non-NotFound error from any underlying read aborts the call and propagates — the caller cannot distinguish "partial result" from "error" so a partial batch is never returned.
Duplicate input names are deduplicated before the round-trip so an N-element caller list with K duplicates issues only N-K reads on the slow path and only N-K OpGet ops on the fast path.
func IsExclusiveCurrency ¶
IsExclusiveCurrency reports whether c is an exclusive (gpu_token-class) currency.
func IsFiniteNonNegAmount ¶
IsFiniteNonNegAmount reports whether x is a usable resource amount: finite (not NaN, not ±Inf) and non-negative. It is the EXACT rule the admission gate applies (currencyAmountValid -> isFiniteNonNeg in resource.go); exporting it lets the API and CLI ingress layers REUSE that single definition so a NaN/Inf amount is rejected at the boundary with a clean 400 / CLI error instead of being accepted and left permanently waiting at admission. A NaN/Inf bypasses the FractionalIntegralCurrencies check (math.Trunc(+Inf)==+Inf is "whole"; NaN>0 is false), so finiteness must be checked on its own. Keeping the rule in ONE place keeps submit-time and admission-time symmetric by construction.
func IsIndexedLabelKey ¶
IsIndexedLabelKey reports whether key is in the physical-index allowlist.
func IsIntegralCurrency ¶
IsIntegralCurrency reports whether c is a counted, indivisible resource for which a positive fractional amount is invalid.
func IsKnownKind ¶
IsKnownKind reports whether a Kind has a registered constructor — a kernel-owned durable kind or a registered extension kind. It is a cheap pre-check a backend uses to reject an unknown kind early (before a List or Watch) rather than failing mid-stream.
func IsWholeNonNegToken ¶
IsWholeNonNegToken reports whether x is the shape an integral-currency OFFER budget (gpu_token, critic_token) must have: finite, non-negative, AND a whole number. An OFFER is the capacity side of the integral-currency invariant — a fractional capacity 0.5 would let two whole-number requests... no: it publishes a budget that the kernel's integral admission gate (which only admits whole-number requests against it) can never let a request consume — a silently DEAD offer. The daemon flag path (cmd/agentosd isWholeNonNeg) already enforces this; exporting it lets the public runtime NodeAgent enforce the same rule from one definition, so an embedded/test caller cannot publish a dead fractional or non-finite integral offer.
func LaneActivityBlockedBackoffActive ¶
LaneActivityBlockedBackoffActive reports whether this in-flight lane is still inside the activity-blocked hold window. ActivityBlocked is an ownership/proof wait, not a retryable activity failure, so it has no DurableRetryState and no retry budget charge.
func LaneHasValidPublishedArtifact ¶
LaneHasValidPublishedArtifact reports whether a lane carries a valid published-artifact stamp: the indexed label is set AND a non-empty artifact reference is present (the label without a ref is a malformed stamp and is treated as absent — recovery must not skip recompute on an unparseable claim). It is the predicate the recovery path uses to choose re-verify over recompute.
func LanePublishedArtifactRef ¶
LanePublishedArtifactRef returns the published artifact's reference, preferring the indexed label and falling back to the ArtifactPublished condition Message (so a stamp written by an older producer that set only the condition still resolves). Empty when the lane has no valid stamp.
func LaneRetryBackoffActive ¶
LaneRetryBackoffActive reports whether this RUNNING lane should not be re-driven until a future NextRetryAt.
func LaneTicketPhaseValue ¶
func LifecycleRedriveCount ¶
LifecycleRedriveCount returns the in-window lifecycle-loss count for the ticket's labels: 0 when none recorded, malformed, or the last loss is older than LifecycleRedriveWindow (decay).
func PathScopeCovers ¶
func PathScopeCovers(scope WriteScope, file string) bool
PathScopeCovers reports whether a repo-relative file is governed by a single path WriteScope: the file equals the scope path or sits beneath it, compared over CANONICAL paths so a `pkg` scope covers `pkg/x.go` but NOT the sibling `pkgx/x.go` (path-SEGMENT containment, not raw string prefix). The same canonicalisation the scope-lease overlap test uses is applied to both sides, so equivalent spellings (`a//b`, `./a/b`, `a/x/../b`) all match `a/b`. A non-path scope (branch / program / benchmark_shard) governs no files and never covers one.
func ProcessHeartbeat ¶
ProcessHeartbeat extracts the last-known heartbeat time from a Process's typed envelope — the LastTransitionTime of its Condition{Type: HeartbeatConditionType}. A Process that has never heart-beaten returns the zero time.Time, whose IsZero() is true.
It is the kernel-owned heartbeat-contract helper: the runtime writer, the reaper controller, and the /proc projector all derive "when did this process last beat" from one function instead of each re-scanning the conditions for a restated string literal. This collapses the heartbeat seam the daemon previously bridged with an ad-hoc adapter.
func Reconcile ¶
func Reconcile(runnable []Ticket, ledger *ResourceLedger) (admitted []Admission, waiting []Ticket)
Reconcile is the pure decision core of the §7.2 Borgmaster loop, simple entry point. Given the runnable tickets and a resource ledger, it returns the tickets it would admit (with their grants) and the tickets it would leave waiting (each with Status.WaitingOn set to the typed currencies that blocked it).
Reconcile is PURE: it CLONES the ledger it is handed and charges the clone, so the caller's ledger is never mutated and calling Reconcile twice with the same ledger pointer yields the same result. (The previous implementation charged the caller-owned ledger in place — repeated calls double-charged and the documented "pure decision core" was not actually pure.)
`effective_target` is emergent: it equals len(admitted) — never a static constant. When demand exceeds capacity the surplus tickets come back in `waiting`, visibly blocked, which is correct backpressure (§2 principle 5).
Reconcile does NOT do cross-tick reservation accounting or preemption — it is the single-tick, runnable-only core. The scheduler loop uses ReconcileColony, which folds running lanes and preemption into the same pass.
func RegisterExclusiveCurrency ¶
func RegisterExclusiveCurrency(c Currency)
RegisterExclusiveCurrency marks an org-declared currency as exclusive (gpu_token-class): held by at most its offered capacity of concurrent holders, refusable at launch time by an out-of-band holder, and therefore subject to contention latching at admission. Call at startup, before the scheduler admits work. It also registers the currency as integral — an exclusive token is by definition counted and indivisible.
func RegisterExecutionResourceCurrency ¶
func RegisterExecutionResourceCurrency(c Currency)
RegisterExecutionResourceCurrency marks an org-declared currency as an execution resource — consumed during a lane's run step, so it is released when the lane reaches its no-further-attempt boundary (vs a scheduling-only currency). Call at startup; the kernel seeds only its built-in infra currencies (provider_token's execution-resource status stays conditional).
func RegisterExtensionKind ¶
RegisterExtensionKind adds a non-kernel durable Kind to the kernel's codec registry so a serialising Store backend can round-trip it. ctor must return a fresh zero-valued Object whose concrete type embeds kernel.ObjectMeta.
It is the additive extension seam (see extensionRegistry): an FC package that introduces its own kernel.Kind (verify, provider, driver) calls this once at init time, e.g.
func init() {
kernel.RegisterExtensionKind(KindVerificationRun,
func() kernel.Object { return &VerificationRun{} })
}
after which kernel.NewObject(KindVerificationRun) returns a typed value and any backend that decodes through NewObject handles the kind. Registering a Kind that is already known (a kernel kind, or an already-registered extension kind) returns ErrKindAlreadyRegistered — a duplicate registration is a programming error, never silently shadowed. ctor must be non-nil.
The constructor's OUTPUT is validated, not just its presence: ctor is called once at registration and its result rejected with ErrBadConstructor if it is nil or a typed-nil. NewObject blindly calls GetMeta() on the constructed object to stamp Kind, so a constructor that returns nil / (*T)(nil) would register fine and later panic the kernel mid-decode. Validating here turns a latent runtime crash into a loud registration-time error.
This is purely additive to the frozen ABI: it adds a new exported function and never changes the behaviour of an existing one. The kernel's own kinds are unaffected — they remain in the closed kindRegistry.
func RegisterIndexedLabelKey ¶
RegisterIndexedLabelKey adds an embedding-owned hot-path label key to the physical secondary-index allowlist. Extension packages call this from init, alongside RegisterExtensionKind. Registration is intentionally additive: kernel-owned labels remain the base contract, and provider/app-specific label names stay in their owning package.
func RegisterIntegralCurrency ¶
func RegisterIntegralCurrency(c Currency)
RegisterIntegralCurrency marks an org-declared currency as counted + indivisible, so a positive fractional request for it is rejected as an exclusivity bypass (the same guard gpu_token has). Call it at startup, before the scheduler admits work — this is how a deployment adds its own integral currency without editing the kernel.
func SchedulerLeaseService ¶
SchedulerLeaseService returns the scheduler service-lease name for a cell — the §15.1 `scheduler/<cell>` queen lease, one leader-elected scheduler per cell, never one global scheduler over a shared ledger.
The DEFAULT cell maps to the bare SchedulerName ("scheduler"), so the single-cell MVP keeps the exact legacy lease name and a caller using SchedulerName and a default-cell Scheduler elect the SAME leader. A named cell maps to "scheduler/<cell>" — the cell-scoped namespace §15 requires.
func SetLaneRetryCondition ¶
func SetLaneRetryCondition(status *Status, state DurableRetryState, exhausted bool)
SetLaneRetryCondition persists retry state into Status.Conditions and keeps WaitingOn operator-visible.
func SetProcessHeartbeat ¶
SetProcessHeartbeat upserts a Process's heartbeat-checkpoint Condition to ts — the write counterpart to ProcessHeartbeat. It updates the in-memory Process struct only; it is NOT a store write. The runtime calls it on Register and on coarse liveness transitions before a (rare) durable checkpoint Update; the per-cadence beat goes to the journal instead (see the HEARTBEAT-CADENCE CONTRACT on HeartbeatConditionType).
func SetProcessMeasuredUsage ¶
func SetProcessMeasuredUsage(p *Process, used ResourceRequest)
SetProcessMeasuredUsage upserts a Process's measured-usage checkpoint Condition to `used` — the write counterpart of ProcessMeasuredUsage. It mutates the in-memory Process struct only; it is NOT a store write. The node-agent/runtime calls it before a (rare) durable checkpoint Update; the per-sample measurement goes to the journal instead (see the cadence contract on MeasuredUsageConditionType). A non-finite or negative amount is dropped so a poisoned measurement cannot reach the ledger via reclamation.
func StampExclusiveContention ¶
StampExclusiveContention writes the contention stamp for the given exclusive currencies onto a ticket's labels at observation time `at`. A caller with no exclusive currencies stamps nothing (the latch only ever derives from a ticket that actually requested an exclusive token). Restamping refreshes the observation time — a re-observed rc=87 re-arms the latch.
func StampLanePublishedArtifact ¶
StampLanePublishedArtifact records that a lane's artifact + required evidence are present, writing both the durable ArtifactPublished condition and its mirrored indexed label (plus the artifact reference). artifactType is a short generic hint (e.g. "git_commit"); artifactRef is the deliverable's reference. A caller with an empty artifactRef stamps nothing — the contract is "a REFERENCED, present artifact", never a bare claim. It is the single writer of the contract so the producer and the recovery consumer cannot disagree.
func TerminalFailureSignature ¶
TerminalFailureSignature is the stable circuit-breaker key for a terminal failure: a short hex hash of the exit code joined with a NORMALISED reason. Normalisation strips the transient detail that makes two instances of the same failure look distinct — digits (iteration/step/PID numbers), filesystem paths, and timestamp-shaped tokens — so repeated-IDENTICAL failures collapse to one signature while a genuinely-different failure gets a new one. It is a pure function so a replay recomputes the same value.
func TerminalLaneOwnedResourceTxnPlan ¶
func TerminalLaneOwnedResourceTxnPlan(store Store, lane *Lane) (LaneOwnedResourceReleaseResult, []Cond, []Op, error)
TerminalLaneOwnedResourceTxnPlan returns the resource-version conditions and delete operations needed to release no-process lane-owned grants and scope leases in the caller's transaction. It deliberately does not discover/delete a Process or provider claims: distributed controllers must pair the returned ops with their own live-owner absence fence before deleting the terminal Lane row. That lets lane-record GC remove the record and any fallback resources in one linearizable transaction instead of proving absence, releasing resources, and then discovering that a process raced in.
func TerminalLaneOwnedResourceTxnPlanWithOptions ¶
func TerminalLaneOwnedResourceTxnPlanWithOptions(store Store, lane *Lane, opts ...TerminalLaneOwnedResourceTxnPlanOption) (LaneOwnedResourceReleaseResult, []Cond, []Op, error)
TerminalLaneOwnedResourceTxnPlanWithOptions is the option-bearing form of TerminalLaneOwnedResourceTxnPlan.
func TerminateRequested ¶
TerminateRequested reports whether an operator terminate signal has been durably recorded on a status — the SignalReceived condition with reason "terminate" and status True. It is the single gate the lane execution path checks before (re)spawning an agent process, so a process death can never race the signal into a fresh attempt: the mark is stamped before the operator expects any effect, and every spawn decision re-reads it.
func TicketNameFromLaneName ¶
TicketNameFromLaneName derives the historical ticket segment from AgentOS lane names for compatibility with pre-TicketName process/claim writers. New code must carry typed TicketName fields; this helper exists only so store normalization can project legacy lane-only ownership into indexed labels.
func TicketPlacementDoneCell ¶
TicketPlacementDoneCell parses a global placement completion marker.
func TicketSchedulingOwnedStatus ¶
TicketSchedulingOwnedStatus reports whether a Ticket status label is owned by kernel scheduling or placement rather than by a desired-state client.
func ValidateEpoch ¶
ValidateEpoch is the §15.1 fencing check for a leader-authored mutating write. currentEpoch is the queen's epoch (from AcquireService or ServiceLeaseEpoch); writeEpoch is the LeadershipEpoch the object/command being written carries. A write whose epoch is BEHIND the current epoch is a stale former leader and is rejected with ErrStaleEpoch. A write at epoch 0 is "no epoch asserted" — it is NOT fenced (purely additive: pre-fencing callers and non-leader writers are unaffected). A write AT or AHEAD of the current epoch is allowed (ahead can happen mid-handoff and self-heals on the next ledger rebuild).
func ValidateTxnPutFencing ¶
ValidateTxnPutFencing enforces the Round-9 admission-#4 ABI contract shared by every Store backend: an OpPut MUST carry a complete object identity (a non-empty UID) OR be paired with a CondResourceVersion on the same object. An OpPut with an empty UID makes a backend INFER the object's identity from the current row; in EtcdStore that inference is a pre-read NOT atomic with the commit, so without a fencing ResourceVersion compare a concurrent delete/recreate could be overwritten under a stale identity. Requiring the RV condition makes the inferred-identity write atomic in BOTH backends. (MemStore would be safe under its mutex, but the ABI the cluster backends share must be identical, so the rule is enforced uniformly here.)
func ValidateTxnScopeOpSet ¶
ValidateTxnScopeOpSet is the shared Store.Txn guard for backends outside pkg/kernel. It rejects one transaction that would create multiple live, mutually-overlapping WriteScopeLease exclusions from the pre-transaction snapshot.
func WriteScopesConflict ¶
func WriteScopesConflict(a, b WriteScope) bool
WriteScopesConflict is the exported, content-free predicate for the canonical write-scope overlap rule used at admission (scopesConflict): two scopes conflict when they are the same lockable kind and overlap — canonical-prefix containment for ScopePath, exact-value equality otherwise. It is exported so callers and tests can assert disjointness against the SAME logic the scheduler enforces, rather than re-deriving path canonicalisation. Pure: no store, no strategy.
Types ¶
type ActivitySummary ¶
type ActivitySummary struct {
// Schema pins the contract id (agentos.agent_activity.v1).
Schema string `json:"schema"`
// Attempt is the Lane.Spec.Attempt the summary was captured for — same
// staleness discipline as TerminalEvidence.Attempt.
Attempt int `json:"attempt,omitempty"`
// Provider/Account/Model/Effort are the ACTUAL spawned identity (from the
// claimed ProviderAccount + route decision), never the ticket's intent.
Provider string `json:"provider,omitempty"`
Account string `json:"account,omitempty"`
Model string `json:"model,omitempty"`
Effort string `json:"effort,omitempty"`
// SessionID is the provider-side session identity the adapter joined
// (e.g. the codex rollout uuid); Source names the adapter.
SessionID string `json:"sessionID,omitempty"`
Source string `json:"source,omitempty"`
// Status is one of the ActivitySummaryStatus* values; UnavailableReason is
// set iff Status is unavailable.
Status string `json:"status"`
StartedAt time.Time `json:"startedAt,omitempty"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
DurationSec float64 `json:"durationSec,omitempty"`
// Cumulative token counters (the provider's own running totals).
TokensInput int64 `json:"tokensInput,omitempty"`
TokensCachedInput int64 `json:"tokensCachedInput,omitempty"`
TokensOutput int64 `json:"tokensOutput,omitempty"`
TokensReasoning int64 `json:"tokensReasoning,omitempty"`
TokensTotal int64 `json:"tokensTotal,omitempty"`
ContextWindow int64 `json:"contextWindow,omitempty"`
// ToolCallsTotal counts every observed tool call; ToolCalls is the bounded
// top-N breakdown (sorted by count desc, then name) with "_other" overflow.
ToolCallsTotal int64 `json:"toolCallsTotal,omitempty"`
ToolCalls []ToolCount `json:"toolCalls,omitempty"`
Turns int `json:"turns,omitempty"`
// Records counts the raw activity records the aggregate was built from;
// Truncated reports the live ring wrapped (the recent-record window lost
// older entries — the AGGREGATE here is still complete).
Records int64 `json:"records,omitempty"`
Truncated bool `json:"truncated,omitempty"`
// Metrics is the bounded last-reading-per-name gauge breakdown an adapter
// observed (sorted by name, capped at ActivitySummaryMetricCap). Progress is
// the last reported position. Both are nil for an LLM-CLI worker that reports
// neither — they exist so a terminal trainer/data-pipeline lane keeps its
// final loss/F1/step on the durable object, queryable post-mortem after the
// live hub ring is gone.
Metrics []MetricGauge `json:"metrics,omitempty"`
Progress *ProgressGauge `json:"progress,omitempty"`
// Rate-limit standing at the last observation (provider used-percent).
RateLimitPrimaryPct *float64 `json:"rateLimitPrimaryPct,omitempty"`
RateLimitSecondaryPct *float64 `json:"rateLimitSecondaryPct,omitempty"`
// CostUSD is set ONLY when an adapter measured a real cost figure — the
// kernel never derives cost from token counts.
CostUSD *float64 `json:"costUSD,omitempty"`
}
ActivitySummary is the bounded, fixed-size durable aggregate of one lane attempt's observed worker activity.
type Admission ¶
type Admission struct {
Ticket Ticket
Grants []ResourceGrant
LaneName string
// Epoch is the LeadershipEpoch the queen held when it admitted this ticket.
// The Spawner stamps it onto the Process and Lane it creates so a row a
// stale queen authored is fenceable (§15.1). It is the same epoch the
// ResourceGrants and WriteScopeLeases for this admission already carry. Zero
// means the admission was produced without a queen epoch (a test / a
// non-leader caller of Reconcile) — additive, never fenced.
Epoch int64
// Cell is the §15.4 cell the admitting queen leads (finding #7). The Spawner
// stamps it onto the Process/Lane via CellLabel so a cell-scoped
// RunningSource folds the lane into the right cell's ledger and never
// another cell's. Empty means DefaultCell — the single-cell MVP.
Cell string
// Fence is the §15.1 fenced-write fence (Round-7 finding #1): the scheduler
// ServiceLease object the admitting queen holds, at the ResourceVersion/Epoch
// it observed this tick. The Spawner routes its correctness-bearing Lane and
// Process creates (and the grant back-stamp) through FencedCreate/
// FencedUpdate with this fence, so a Lane/Process a STALE queen authored
// atomically fails to write — closing the check-then-write TOCTOU gap a bare
// pre-write epoch re-read leaves open. A zero-valued Fence (no Kind/Name) is
// a non-leader / test admission; the Spawner then falls back to plain writes.
Fence FenceRef
}
Admission pairs a Ticket the scheduler admitted in a tick with the ResourceGrants admission issued for it. The loop wrapper turns each Admission into a spawned Lane (a journaled Process row first).
type AdmissionGate ¶
type AdmissionGate interface {
// Prepare refreshes the gate's per-tick snapshot. The scheduler loop calls
// it ONCE per tick BEFORE the pure decision core, so I/O is allowed here. It
// must be bounded and must not panic; an internal error leaves the previous
// snapshot in place (fail-open).
Prepare()
// HoldReason returns a non-empty typed reason to DEFER (hold) a ticket this
// tick instead of admitting it. It is PURE and FAIL-OPEN: it does no I/O,
// evaluates only the snapshot Prepare built, and returns "" to admit. It is
// called from the pure decision core, so it must be cheap and deterministic
// with respect to the returned reason.
HoldReason(t Ticket) string
// OnDecision is called ONCE per tick AFTER the decision with every waiting
// ticket (gate-held and otherwise), so the gate can materialize durable side
// effects — e.g. open a human-escalation object for a ticket it held — OUTSIDE
// the pure core. I/O is allowed; it must be idempotent (it runs every tick a
// ticket stays held) and must not panic.
OnDecision(waiting []Ticket)
}
AdmissionGate is the optional pre-admission hook. Its lifecycle within one scheduler tick is Prepare (once, may do I/O) → HoldReason (per runnable ticket, pure) → OnDecision (once, may do I/O).
type App ¶
type App struct {
ObjectMeta `json:",inline"`
Spec AppSpec `json:"spec"`
Status Status `json:"status"`
}
App is an installable workflow application; AppInstall is its install receipt. Both follow the manifest ABI.
func (*App) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the App.
type AppInstall ¶
type AppInstall struct {
ObjectMeta `json:",inline"`
Spec AppInstallSpec `json:"spec"`
Status Status `json:"status"`
}
AppInstall is the install receipt for an App on a Node.
func (*AppInstall) DeepCopyObject ¶
func (a *AppInstall) DeepCopyObject() Object
DeepCopyObject returns an independent copy of the AppInstall.
func (*AppInstall) GetMeta ¶
func (a *AppInstall) GetMeta() *ObjectMeta
GetMeta makes AppInstall an Object.
type AppInstallSpec ¶
type AppInstallSpec struct {
AppName string `json:"appName"`
NodeName string `json:"nodeName"`
InstalledAt time.Time `json:"installedAt"`
}
AppInstallSpec is the desired-state of an AppInstall.
type AppendStats ¶
type AppendStats struct {
// Count is the number of Append calls observed.
Count int64 `json:"count"`
// TotalLatency is the summed wall-clock spent inside Append.
TotalLatency time.Duration `json:"totalLatency"`
// MeanLatency is TotalLatency / Count (0 when Count is 0).
MeanLatency time.Duration `json:"meanLatency"`
// MaxLatency is the slowest single Append observed.
MaxLatency time.Duration `json:"maxLatency"`
}
AppendStats is a snapshot of the journal's Append-latency instrumentation.
type BatchedGetter ¶
type BatchedGetter interface {
// GetMany returns one map entry per requested name that exists. A name
// not present in the store yields no map entry — the same shape callers
// already handle when Store.Get returns ErrNotFound. The kind is fixed
// per call; mixing kinds in one batch is intentionally out of scope so
// the backend can issue ONE per-kind range or Txn op group without
// per-element kind decoding.
//
// Duplicate names in the input are coalesced (one entry per unique name
// in the output). An empty or nil names slice returns an empty map with
// no backend round-trip, mirroring how the existing per-key loops noop
// on empty input.
//
// Returned objects follow the same independent-copy contract as Store.Get:
// the caller may mutate them freely without touching the store state.
GetMany(kind Kind, names []string) (map[string]Object, error)
}
BatchedGetter is an optional Store extension that returns many objects of one Kind in a single round-trip. The kernel ABI's Store.Get is one-key-per-call; the etcd backend implements this interface by issuing a single Txn with N OpGet ops, collapsing what would otherwise be N sequential RPCs (each gated by network + WAL latency) into one round-trip.
The scheduler running-source rebuild is the original hot path: it loads N processes, then for each process loads ~2-4 grants, 1 ticket, optionally 1 lane. With ~10 processes × ~5 GETs = 50 sequential RPCs at 200ms/round-trip observed under load (etcd slow-query log, 2026-05-26), that was 10s of every scheduler tick — leases (TTL 15s) frequently expired before the next renewal completed. Replacing the inner loop with one GetMany per kind drops the same work to 3 Txn round-trips.
A backend that does NOT implement BatchedGetter is correct: callers route through the free function GetMany below, which falls back to per-key Get and returns the same map shape. The optimisation is opt-in per backend, not a requirement of the Store ABI.
type Cell ¶
type Cell struct {
ObjectMeta `json:",inline"`
Spec CellSpec `json:"spec"`
Status Status `json:"status"`
}
Cell is the §15.4 unit of consistency and scale. One cell owns one linearizable resource ledger, one Process authority, one write-scope-lease namespace, and one scheduler (one queen, the `scheduler/<cell>` lease). Scale is more cells, never sharding one scheduler across a shared ledger. This is a minimal first-class object: it gives cell-scoped leases and quotas a durable home; a thin global placer (a later phase) writes Cell rows and assigns ticket/budget/scope affinity to them.
func (*Cell) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the Cell.
type CellDirectory ¶
type CellDirectory struct {
ObjectMeta `json:",inline"`
Spec CellDirectorySpec `json:"spec"`
Status Status `json:"status"`
}
func PublishCellDirectory ¶
func PublishCellDirectory(ctx context.Context, store Store, opts CellDirectoryPublishOptions) (*CellDirectory, error)
PublishCellDirectory pages Cell discovery off the CellPlacer activation, stamps freshness, and upserts the bounded directory object the placer reads by Get during Tick. DefaultCell is excluded because it is never a placement target.
func (*CellDirectory) DeepCopyObject ¶
func (d *CellDirectory) DeepCopyObject() Object
func (*CellDirectory) GetMeta ¶
func (d *CellDirectory) GetMeta() *ObjectMeta
type CellDirectoryEntry ¶
type CellDirectoryEntry struct {
Name string `json:"name"`
ResourceVersion int64 `json:"resourceVersion,omitempty"`
Generation int64 `json:"generation,omitempty"`
}
CellDirectoryEntry is one Cell object discovered off the placer lease path.
type CellDirectorySpec ¶
type CellDirectorySpec struct {
Cells []CellDirectoryEntry `json:"cells"`
MeasuredAt time.Time `json:"measuredAt"`
}
type CellPlacementResult ¶
CellPlacementResult reports one placement tick.
type CellPlacer ¶
type CellPlacer struct {
// contains filtered or unexported fields
}
CellPlacer is the single global placement authority. It is deliberately much narrower than the scheduler: it never reads offers, reserves resources, issues grants, or spawns lanes. Its global-store writes are fenced marker transitions: triage->placed-to-<cell> for intent, then copied-to-<cell> after the target store confirms the deterministic copy exists.
func NewCellPlacer ¶
func NewCellPlacer(cfg CellPlacerConfig) *CellPlacer
NewCellPlacer builds a placement authority. A nil Store falls back to the LeaseManager's store, matching NewScheduler's single-store default.
func (*CellPlacer) Tick ¶
func (p *CellPlacer) Tick() CellPlacementResult
Tick runs one placement pass. It first wins or renews the placer ServiceLease; a non-leader returns HeldLease=false and makes no writes. While leader, it completes any crash-left global placement markers, then hashes each unassigned/default triage ticket onto the current live named Cell set.
type CellPlacerConfig ¶
type CellPlacerConfig struct {
Holder string
Store Store
Leases *LeaseManager
CellStore CellStoreResolver
LeaseTTL time.Duration
CopiedRetention time.Duration
LiveCells []string
CellDirectoryName string
CellDirectoryFreshness time.Duration
}
CellPlacerConfig wires the leased placement authority.
type CellSpec ¶
type CellSpec struct {
// CellID is the stable cell identity used in the `scheduler/<CellID>`
// scheduler-lease name and every cell-scoped lease namespace.
CellID string `json:"cellID"`
// Zone, Region, and Cloud describe the cell as a placement and failure
// domain. Empty values preserve the single-cell local mode.
Zone string `json:"zone,omitempty"`
Region string `json:"region,omitempty"`
Cloud CloudProvider `json:"cloud,omitempty"`
// StoreEndpoint is the optional per-cell store endpoint for future etcd
// isolation. Empty means the current shared store.
StoreEndpoint string `json:"storeEndpoint,omitempty"`
// ProviderTokenBudget is the per-cell budget of the globally scarce
// provider_token currency (§15.4: truly global resources are budgeted per
// cell and rebalanced periodically, never synchronously shared). Zero means
// "unbudgeted" until the global placer assigns one.
ProviderTokenBudget float64 `json:"providerTokenBudget,omitempty"`
}
CellSpec is the desired-state of a Cell.
type CellStoreResolver ¶
CellStoreResolver resolves the object store for a named Cell. The global CellPlacer is the only cross-store actor: it reads tickets from the global entry queue, CASes a placement marker there, and copies the ticket into the target cell's store for single-store admission by that cell's queen.
type Clock ¶
Clock returns the current time. It is an injectable seam so lease-expiry tests do not have to sleep.
type CloudProvider ¶
type CloudProvider string
CloudProvider names the cloud a Node or Cell belongs to. Empty is valid and means topology is unspecified, preserving local single-node mode.
const ( CloudProviderGCP CloudProvider = "gcp" CloudProviderAWS CloudProvider = "aws" CloudProviderAzure CloudProvider = "azure" CloudProviderLocal CloudProvider = "local" )
type Cond ¶
type Cond struct {
Type CondType
Kind Kind
Name string
ResourceVersion int64
Fence FenceRef
Scope WriteScope
Cell string
Holder string
At time.Time
LabelKey string
LabelValue string
}
Cond is one Store.Txn condition. Object conditions use Kind/Name; fence conditions use Fence; scope conditions use Scope/Cell/Holder/At.
type CondType ¶
type CondType string
CondType names one predicate in a Store.Txn precondition list.
const ( // CondResourceVersion requires the object to exist at ResourceVersion. CondResourceVersion CondType = "resource_version" // CondExists requires the object to exist. CondExists CondType = "exists" // CondNotExists requires the object to be absent. CondNotExists CondType = "not_exists" // CondFence applies a FenceRef: by default resource version plus optional // epoch/instance identity; with IdentityOnly, stable ServiceLease // InstanceID+Epoch. Lease wall-clock liveness is an acquire/renew/release // discipline, not the commit-time safety boundary. CondFence CondType = "fence" // CondScopeAvailable requires no live overlapping WriteScopeLease in Cell. CondScopeAvailable CondType = "scope_available" // CondNoIndexedObjects requires an indexed exact-label query to return no // objects. It is for distributed absence fences such as "no Process owns // ticket X" where a prior List proof would otherwise race a concurrent // creator before the transaction commits. CondNoIndexedObjects CondType = "no_indexed_objects" )
type Condition ¶
type Condition struct {
Type string `json:"type"`
Status ConditionStatus `json:"status"`
Reason string `json:"reason,omitempty"`
Message string `json:"message,omitempty"`
LastTransitionTime time.Time `json:"lastTransitionTime"`
}
Condition is one observed aspect of an object's state. The Type is a kind-specific predicate (e.g. "Admitted", "Ready"); Status carries its current truth; Reason/Message explain a transition for /proc and audit.
type ConditionStatus ¶
type ConditionStatus string
ConditionStatus is the tri-state truth value of a Condition, following the Kubernetes convention.
const ( ConditionTrue ConditionStatus = "True" ConditionFalse ConditionStatus = "False" ConditionUnknown ConditionStatus = "Unknown" )
type ConflictError ¶
ConflictError is the typed CAS-conflict error. It carries the expected and actual ResourceVersion so a caller can log the race precisely. It wraps ErrConflict, so errors.Is(err, ErrConflict) is true.
func (*ConflictError) Error ¶
func (e *ConflictError) Error() string
func (*ConflictError) Unwrap ¶
func (e *ConflictError) Unwrap() error
Unwrap lets errors.Is(err, ErrConflict) succeed for a *ConflictError.
type ConsumerFailureRecord ¶
type ConsumerFailureRecord struct {
Attempts int
DeadLettered bool
NewlyDeadLettered bool
DeadLetterOffset int64
AlreadyCommitted bool
}
ConsumerFailureRecord is the journal-owned retry/dead-letter state for one failed delivery of a source event to a consumer group.
type Currency ¶
type Currency string
Currency names one dimension of measured capacity. Borg's lesson (§7.3): one resource model, several explicit currencies — never lane-slots in one component and a fictional ram_mb in another. Admission gates against ALL currencies simultaneously.
const ( // CurrencyWorkspaceSlot is a warm, validated workspace slot. CurrencyWorkspaceSlot Currency = "workspace_slot" // CurrencyRAMBytes is host RAM in bytes. CurrencyRAMBytes Currency = "ram_bytes" // CurrencyCPU is CPU capacity in whole cores (fractional amounts allowed). CurrencyCPU Currency = "cpu" // CurrencyProviderToken is a unit of per-account provider concurrency. CurrencyProviderToken Currency = "provider_token" // CurrencyScopeLease is a write-scope lease slot. CurrencyScopeLease Currency = "scope_lease" // CurrencyGPUToken is one exclusive local accelerator (the M5 Max GPU). A // node that has the GPU offers capacity 1; a workload that needs it (model // training) requests gpu_token:1, so admission's per-currency gate // serializes training to one concurrent run and surfaces the wait in /proc. // See docs/architecture/training-as-agentos-workload-EDD.md (gap G1). CurrencyGPUToken Currency = "gpu_token" // CurrencyStoreWriteOps models the single-writer substrate (etcd // write-throughput-PER-TICK) as a first-class admittable currency — the REAL // bottleneck behind the 7-10-useful-lane cap (Principle 3 / G3-a). The scheduler // stops admitting once a tick's cumulative bind fan-out (admissionTxnOpEstimate) // would saturate the single etcd writer, surfacing the wait as `store_write_ops` // in /proc/capacity instead of silently capping. Unlike the persistent currencies // (ram/slots/gpu) it is consumed-and-RESET each tick — a running lane carries no // reservation forward (see chargeStoreWriteOps). Capacity is the measured // sustainable writer rate as a per-tick offer; an absent offer means "unmetered" // (unlimited), so the gate is opt-in and never throttles an unmeasured writer. CurrencyStoreWriteOps Currency = "store_write_ops" // CurrencyMPSExclusive marks a Work as holding the node's single GPU/MPS engine // EXCLUSIVELY. Unlike the counted CurrencyGPUToken, the kernelv2 intake bridge // maps it to WorkSpec.Resources.MPSExclusive, where the kernelv2 admission gate // refuses to co-admit it while ANY live mps-exclusive holder exists (including an // observed trainer the kernel only sees via the observe plane). Generic facet. CurrencyMPSExclusive Currency = "mps_exclusive" // CurrencyMemHeavy marks a Work as a RAM-HEAVY node holder (a heavy 9B trainer or a // big-corpus curation pass). Like CurrencyMPSExclusive, the kernelv2 intake bridge maps // it to WorkSpec.Resources.MemHeavy, where the admission gate refuses to co-admit it // while ANY live heavy holder exists (kernel-spawned OR observed) — the swap-thrash // guard. Orthogonal to mps (a GPU trainer declares both; RAM-heavy CPU/IO declares // mem_heavy only). Generic facet (the M5 is one such node — equally a CUDA/EKS node). CurrencyMemHeavy Currency = "mem_heavy" )
The kernel's resource currencies.
func ExclusiveCurrenciesOf ¶
func ExclusiveCurrenciesOf(req ResourceRequest) []Currency
ExclusiveCurrenciesOf returns the exclusive currencies a request names with a positive amount, sorted lexically (deterministic stamps). Nil when none.
type Decision ¶
type Decision struct {
Admitted []Admission
Waiting []Ticket
Preempted []Preemption
}
Decision is the full output of the colony decision core: the tickets to admit (with grants), the tickets left waiting (each with a typed WaitingOn), and the running lanes to preempt to make room for higher-priority work.
func ReconcileColony ¶
func ReconcileColony(input ReconcileInput) Decision
ReconcileColony is the full §15 colony decision core: a single PURE pass that (1) folds every running lane's reservation back into a CLONE of the ledger so already-running work reduces this tick's headroom (cross-tick accounting — the v1-class over-admission fix), (2) iterates runnable tickets in priority order calling TryAdmit and charging the clone between admissions, (3) when a higher-priority ticket does not fit, selects the lowest-priority preemptible running lane whose release would help and records a Preemption, freeing its reservation back to the ledger so the blocked ticket can then be admitted.
It is pure: the caller's ledger is cloned, so repeated calls are deterministic. Every issued ResourceGrant is stamped with input.Epoch and a durable-stable name.
type DurableRetryState ¶
type DurableRetryState struct {
Attempts int `json:"attempts"`
FirstSeenAt time.Time `json:"firstSeenAt"`
LastAttemptAt time.Time `json:"lastAttemptAt"`
NextRetryAt time.Time `json:"nextRetryAt"`
Reason string `json:"reason"`
}
DurableRetryState is the durable retry ledger shared by retryable lane execution and bounded controller reconciliation.
func LaneRetryStateFromStatus ¶
func LaneRetryStateFromStatus(status Status) (DurableRetryState, bool)
LaneRetryStateFromStatus reads durable retry state from a Lane Status.
type Event ¶
type Event struct {
Offset int64 `json:"offset"`
Type EventType `json:"type"`
Subject string `json:"subject,omitempty"` // object Name the event concerns
Payload map[string]any `json:"payload,omitempty"`
Timestamp time.Time `json:"timestamp"`
}
Event is one append-only journal entry. Offset is assigned by the journal on Append and is monotonic and gap-free per journal; consumer groups commit against it. Payload is an opaque kind-specific map.
type EventType ¶
type EventType string
EventType classifies a journal entry. The journal is the high-churn log (§7.7): process status, lane progress, telemetry, heartbeats — never the durable object store.
const ( EventWorkAvailable EventType = "WorkAvailable" EventAdmission EventType = "Admission" EventLaneProgress EventType = "LaneProgress" EventRework EventType = "Rework" EventEscalation EventType = "Escalation" EventHeartbeat EventType = "Heartbeat" EventProcessStatus EventType = "ProcessStatus" EventTelemetry EventType = "Telemetry" EventDeadLetter EventType = "DeadLetter" // EventSchedulerError records a reconcile tick that failed for a reason // other than lease loss (a store/backend outage, a runnable-read failure, // a spawn failure). The supervision tree consumes it: a real scheduler // outage must be visible, never swallowed behind fail-closed behaviour. EventSchedulerError EventType = "SchedulerError" // EventDeliberationTurn is the durable dissent-ledger record of one // multi-agent deliberation turn (public vs private position, held-back // concerns, engagement). hive.Producers.PublishDeliberationTurn appends it // alongside the lossy MsgDeliberationTurn gossip hint so the org-health // record survives a dropped message and a consumer-group replay can rebuild // the deliberation. Advisory only — it never gates a verdict. EventDeliberationTurn EventType = "DeliberationTurn" )
The journal event types the kernel emits.
const EventBatchConverged EventType = "BatchConverged"
EventBatchConverged is the journal event type a fan-in §15.6 Wait (WaitBatchConverged) is satisfied by: every child of a batch reached a terminal state. It is appended durably by SatisfyCondition before any waiter is notified, so the reconcile backstop can always find it.
const EventExclusiveIdleHole EventType = "ExclusiveIdleHole"
EventExclusiveIdleHole is the typed journal event for a persistent exclusive-currency admission hole. Payload: currency, idle_for_seconds, waiting (bounded ticket-name sample), waiting_count, severity ("high"), reason ("exclusive_currency_admission_hole"), leadership_epoch.
type ExternalResourceReleaseFunc ¶
ExternalResourceReleaseFunc is an optional application/provider-layer cleanup hook for non-kernel resources tied to a lane. The kernel passes the Store, lane name, and optional extra conditions; the owning package decides object kinds, claim names, and proof conditions. Extra conditions carry the caller's activation fence when cleanup must be atomic with a scheduler/controller lease.
type FenceRef ¶
type FenceRef struct {
// Kind and Name locate the fence object — for the admission transaction,
// KindServiceLease and the scheduler lease's object Name.
Kind Kind
Name string
// ResourceVersion is the fence object's version the queen observed. The
// write commits only if the store still holds the fence at exactly this
// version.
ResourceVersion int64
// Epoch is the LeadershipEpoch the fence object must still carry — the
// §15.1 fencing token. Zero means "do not check the epoch" (a non-leader /
// test caller); a real queen always passes a positive epoch.
Epoch int64
// InstanceID is the per-acquisition lease identity the fence object must
// still carry. Empty means "do not check instance identity".
InstanceID string
// IdentityOnly fences a ServiceLease by InstanceID + Epoch, ignoring
// ResourceVersion. It is for actor/control writes whose own renewal loop
// legitimately bumps ResourceVersion. Scheduler admission and scope fencing
// leave this false to retain the exact object-version fence.
IdentityOnly bool
}
FenceRef identifies the object a fenced write is conditioned on — normally the scheduler ServiceLease — and the exact state that object must still hold for the write to commit. In the default resource-version mode, the fence is satisfied ONLY when the stored fence object is present AND both:
- its ResourceVersion equals ResourceVersion (it has not been written since the queen read it — covers a successor re-acquire / a release that mutated the lease row), and
- its LeadershipEpoch equals Epoch (defence-in-depth: a successor advances the epoch on take-over, so an epoch mismatch is also a stale leader even if a ResourceVersion check were somehow bypassed).
IdentityOnly is an additive mode for actor activations: it fences a ServiceLease by stable acquisition identity (InstanceID + Epoch) rather than by ResourceVersion. A self-renew rewrites the lease row and bumps ResourceVersion but preserves that identity, so it must not invalidate an in-flight actor write. A release or different-instance take-over changes the identity and fails closed.
SAFETY BOUNDARY. ServiceLease fence safety is single-writer via identity, not exact wall-clock liveness. The wall-clock deadline is the mechanism that lets a successor attempt takeover. A successor only succeeds by atomically changing the ServiceLease InstanceID+Epoch (under MemStore's mutex or in one etcd transaction including the auxiliary identity key), so a predecessor's in-flight fenced write either commits before the successor has acquired or fails after the identity changes. The bounded interval after RenewedAt+TTL but before an etcd native lease has asynchronously deleted the key is therefore single-writer-safe when no successor has acquired.
The selected conditions are checked atomically with the write. A FenceRef with a zero Kind/Name is a programming error — FencedCreate/FencedUpdate reject it.
func (FenceRef) Valid ¶
Valid reports whether the FenceRef names a fence object at all. A fenced write with no fence is a caller bug — the primitive exists to bind a write to a fence. A FenceRef that is NOT Valid (its zero value) signals "no leadership asserted": a non-leader / test caller, for which the fenced-write callers (the admission transaction, the Spawner) fall back to a plain Create/Update.
type FencedWriter ¶
type FencedWriter interface {
// FencedCreate creates obj iff the fence still holds. See the interface doc.
FencedCreate(obj Object, fence FenceRef) (Object, error)
// FencedUpdate compare-and-swaps obj iff the fence still holds. See the
// interface doc.
FencedUpdate(obj Object, fence FenceRef) (Object, error)
}
FencedWriter is the ADDITIVE store-ABI extension a backend implements to offer genuinely atomic fenced writes (§15.1, Round-7 finding #1). It is OPTIONAL on the Store interface — the frozen Store contract is unchanged — but every production backend implements it; FencedCreate / FencedUpdate fail closed (ErrFenceUnsupported) on a backend that does not.
The contract of both methods: the object is created / updated ONLY IF the FenceRef's fence object still satisfies its selected mode, evaluated ATOMICALLY with the write. There is no window between the fence check and the write — they are one store operation. A stale fence yields ErrFenceStale and nothing is written.
- FencedCreate behaves like Create (ErrAlreadyExists on a duplicate) once the fence passes.
- FencedUpdate behaves like Update (a *ConflictError on a stale ResourceVersion, ErrNotFound on an absent object) once the fence passes.
The fence check is evaluated FIRST: a stale fence is reported as ErrFenceStale even if the write itself would also have failed.
type ForcedDropRecord ¶
type ForcedDropRecord struct {
// Group is the consumer group whose cursor was force-advanced.
Group string `json:"group"`
// FromCursor is the group's committed cursor before the forced advance.
FromCursor int64 `json:"fromCursor"`
// ToCursor is the retention floor the cursor was advanced to.
ToCursor int64 `json:"toCursor"`
// Dropped is the number of un-acked events skipped (ToCursor - FromCursor).
Dropped int64 `json:"dropped"`
// MarkerOffset is the offset of the EventDeadLetter audit marker recording
// this forced drop.
MarkerOffset int64 `json:"markerOffset"`
}
ForcedDropRecord describes one consumer group whose cursor was force-advanced past the retention floor because its lag exceeded the force-drop horizon. It is returned for observability/audit; the durable record is the appended EventDeadLetter marker.
type Journal ¶
type Journal interface {
Append(Event) (int64, error)
Subscribe(group string) error
Unsubscribe(group string) error
Poll(group string) ([]Event, error)
Commit(group string, cursor int64) error
CommittedCursor(group string) (int64, error)
Lag(group string) (int64, error)
Head() (int64, error)
}
Journal is the append-only event log interface consumed by the kernel, workflow engine, Hive, API surfaces, and /proc. It is the Kafka/Flink primitive of §7.7: producers Append immutable events; consumer groups Subscribe, Poll, and Commit durable cursors; readers observe Head and Lag.
Implementations include MemJournal for local/dev/tests and pkg/etcdstore.EtcdJournal for production durability.
type JournalRetentionStats ¶
type JournalRetentionStats struct {
// Head is the offset the next Append will use.
Head int64 `json:"head"`
// Base is the oldest retained offset (compaction has trimmed below it).
Base int64 `json:"base"`
// LiveLen is the retained live-log length (Head - Base).
LiveLen int64 `json:"liveLen"`
}
JournalRetentionStats is a read-only snapshot of the journal's retained-size state for /proc. LiveLen is the measured-truth journal-growth signal H3 must keep bounded; Head/Base bracket the retained offset window.
type JournalRetentionStatsReader ¶
type JournalRetentionStatsReader interface {
RetentionStats() (JournalRetentionStats, error)
}
JournalRetentionStatsReader is the optional read-only Journal extension /proc uses to surface journal size alongside per-consumer lag. Both MemJournal and etcdstore.EtcdJournal implement it; a journal that does not simply omits the size fields from the /proc view.
type Kind ¶
type Kind string
Kind names the type of a kernel object. Every durable object declares its Kind in the envelope so the store can index and route it.
const ( // KindCellDirectory is the bounded live-cell discovery snapshot consumed // by the global CellPlacer under its ServiceLease. KindCellDirectory Kind = "CellDirectory" // DefaultCellDirectoryName is the deterministic key for the global cell // directory snapshot. DefaultCellDirectoryName = "live-cells" )
const ( KindTicket Kind = "Ticket" KindLane Kind = "Lane" KindProcess Kind = "Process" KindServiceLease Kind = "ServiceLease" KindWriteScopeLease Kind = "WriteScopeLease" KindResourceGrant Kind = "ResourceGrant" KindApp Kind = "App" KindAppInstall Kind = "AppInstall" KindNode Kind = "Node" // KindCell is the §15.4 unit of consistency and scale: one cell owns one // linearizable resource ledger, one Process authority, one write-scope-lease // namespace, and one scheduler (one queen). It is a minimal first-class // object so cell-scoped leases and quotas have a durable home. KindCell Kind = "Cell" )
The durable object kinds. These live in the object store (etcd in production, MemStore in tests); high-churn status/telemetry lives in the journal instead.
type Lane ¶
type Lane struct {
ObjectMeta `json:",inline"`
Spec LaneSpec `json:"spec"`
Status Status `json:"status"`
}
Lane is a durable execution of a Ticket — the workflow instance. It holds the journal cursor (where replay resumes) and the lane lifecycle state.
func (*Lane) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the Lane.
type LaneOwnedResourceReleaseOption ¶
type LaneOwnedResourceReleaseOption func(*laneOwnedResourceReleaseConfig)
LaneOwnedResourceReleaseOption tunes ReleaseLaneOwnedResources.
func WithLaneOwnedResourceExternalRelease ¶
func WithLaneOwnedResourceExternalRelease(fn ExternalResourceReleaseFunc) LaneOwnedResourceReleaseOption
WithLaneOwnedResourceExternalRelease installs the caller-owned cleanup hook for non-kernel lane resources. The kernel does not interpret the object's kind or naming scheme.
func WithLaneOwnedResourceFence ¶
func WithLaneOwnedResourceFence(fence FenceRef) LaneOwnedResourceReleaseOption
WithLaneOwnedResourceFence routes cleanup writes through the same scheduler/controller ServiceLease fence as the lane resource writer. When no explicit writer is passed, grant/scope/process writes are executed as Store.Txn operations conditioned on this fence and the target object's ResourceVersion.
func WithLaneOwnedResourceSkipExternalRelease ¶
func WithLaneOwnedResourceSkipExternalRelease() LaneOwnedResourceReleaseOption
WithLaneOwnedResourceSkipExternalRelease leaves application/provider-layer resources untouched. Callers that use transaction-time active-resource absence fences can choose to let the owning controller release them instead of racing cleanup against a freshly-created resource for the same lane.
func WithLaneOwnedResourceSkipProcessDiscovery ¶
func WithLaneOwnedResourceSkipProcessDiscovery() LaneOwnedResourceReleaseOption
WithLaneOwnedResourceSkipProcessDiscovery prevents cleanup from discovering and deleting a Process when the caller has already proven that no process may own the lane. GC paths use this with a transaction-time Process absence fence on the final lane delete, so a raced-in process blocks deletion instead of being swept up as cleanup.
func WithLaneOwnedResourceTrimProviderToken ¶
func WithLaneOwnedResourceTrimProviderToken() LaneOwnedResourceReleaseOption
WithLaneOwnedResourceTrimProviderToken permits provider_token to be removed from a ResourceGrant after the caller has supplied proof through an external release hook or an equivalent absence fence. The currency is still generic scheduler capacity; the provider package owns claim lifecycle proof.
func WithLaneOwnedResourceWriter ¶
func WithLaneOwnedResourceWriter(writer LaneResourceWriter) LaneOwnedResourceReleaseOption
WithLaneOwnedResourceWriter routes cleanup writes through writer. Controllers pass an activation-fenced writer; scheduler/kernel callers omit it and get CAS deletes plus ordinary CAS updates.
type LaneOwnedResourceReleaseResult ¶
type LaneOwnedResourceReleaseResult struct {
ProcessDeleted bool
GrantsDeleted int
GrantsTrimmed int
ScopeLeasesReleased int
ExternalReleaseTried int
ExternalReleaseFreed int
NoProcessGrantDelete int
}
LaneOwnedResourceReleaseResult is measured cleanup work from one idempotent lane-owned resource release pass.
func ReleaseLaneExecutionResources ¶
func ReleaseLaneExecutionResources(store Store, lane *Lane, proc *Process, opts ...LaneOwnedResourceReleaseOption) (LaneOwnedResourceReleaseResult, error)
ReleaseLaneExecutionResources frees local execution capacity after a lane's workflow has reached a point that cannot run another agent attempt, while leaving the lane's Process row and metadata intact for merge and terminal cleanup. Terminal cleanup debt is still capacity-accounted, but scope leases are not re-renewed by that debt. provider_token is freed only when the caller supplies external lifecycle proof through options; the kernel does not know provider claim object names or states.
func ReleaseLaneOwnedResources ¶
func ReleaseLaneOwnedResources(store Store, lane *Lane, proc *Process, opts ...LaneOwnedResourceReleaseOption) (LaneOwnedResourceReleaseResult, error)
ReleaseLaneOwnedResources is the single durable cleanup primitive for a lane's Process row, ResourceGrants, WriteScopeLeases, and any caller-supplied external resources. It works from durable ownership records: Process.Spec.GrantNames and Process.Spec.ScopeLeases when a Process exists, with lane/ticket epoch fallbacks for pre-process recovery. ErrNotFound is idempotent success; CAS or fence conflicts are returned so the caller retries the whole primitive on the next pass.
type LaneResourceWriter ¶
LaneResourceWriter is the CAS/fenced write surface used by lane resource cleanup. Controllers pass an activation-fenced writer; kernel paths use the default DeleteCAS/Update writer below.
type LaneSpec ¶
type LaneSpec struct {
TicketName string `json:"ticketName"`
// ScopeLeases are the exact WriteScopeLease acquisitions created by the
// admission transaction for this lane. The Process copies them later, but
// the Lane copy is atomic with scope acquisition and survives a crash before
// Process creation.
ScopeLeases []WriteScopeLeaseIdentity `json:"scopeLeases,omitempty"`
// JournalCursor is the offset in the journal this lane's workflow has
// durably consumed; replay after a crash resumes here.
JournalCursor int64 `json:"journalCursor"`
// Attempt counts agent runs; the workflow caps it at MaxLaneAttempts.
Attempt int `json:"attempt"`
}
LaneSpec is the desired-state of a Lane.
type LaneState ¶
type LaneState string
LaneState is the Aurora-clean lane state machine value.
const ( LanePending LaneState = "PENDING" LaneAdmitted LaneState = "ADMITTED" LaneRunning LaneState = "RUNNING" LaneVerifying LaneState = "VERIFYING" LaneReworking LaneState = "REWORKING" LaneDone LaneState = "DONE" LaneEscalated LaneState = "ESCALATED" LanePreempted LaneState = "PREEMPTED" LaneLost LaneState = "LOST" )
type LeaseFencing ¶
LeaseFencing is the full §15.1 fencing identity of a service lease at one instant: the LeadershipEpoch, whether the lease is still LIVE (RenewedAt+TTL in the future), and the immutable per-acquisition InstanceID. The three together are what a stale-leader fence needs — an EXPIRED or RELEASED lease can still carry the same Epoch number a queen remembers, so the epoch alone is not proof of leadership; Live and InstanceID close that gap.
type LeaseManager ¶
type LeaseManager struct {
// contains filtered or unexported fields
}
LeaseManager is the leader-election + write-scope-lock primitive. It is backed by a Store, so leases are durable objects.
A ServiceLease is a single object, so its acquire is race-safe purely on the store's CAS: 100 concurrent acquirers all attempt the same Create/Update and the compare-and-swap admits exactly one (§11 duplicate-leader SLO).
FENCING (§15.1). TTL + CAS alone is NOT enough to fence a stale leader: a former queen that paused past its TTL, then resumed, would still pass the CAS on every later write because each write does its own fresh Get and so always carries the current ResourceVersion. Two additive fields close that:
- Epoch — a monotonic LeadershipEpoch, incremented on every fresh take-over. The queen stamps it on every correctness-bearing object; ValidateEpoch rejects a write whose epoch is behind the current one.
- InstanceID — an immutable per-acquisition token. RenewServiceInstance and ReleaseServiceInstance authenticate against it, so a stale process reusing the same holder STRING cannot renew or release its successor's lease.
A WriteScopeLease acquire is a check-then-act across MANY objects. Stores that implement ScopeConflictGuard own that overlap scan and lease write as one backend-atomic operation: MemStore does it under its store mutex, and the etcd backend does it in one transaction. scopeMu remains only as a legacy fallback for Store wrappers that do not expose the guard.
func NewLeaseManager ¶
func NewLeaseManager(store Store, clock Clock) *LeaseManager
NewLeaseManager returns a LeaseManager over the given Store. Pass nil for clock to use the system clock.
When the store is a *MemStore and an explicit (test) clock is supplied, the MemStore's own clock is set to the SAME clock. MemStore-backed lease helpers use that clock for acquire/renew/release and write-scope expiry, so a LeaseManager and the MemStore it leases over must agree on time. Production passes a nil clock — both default to the system clock and already agree — so this only binds the two together for a deterministic test clock.
func (*LeaseManager) AcquireScope ¶
func (lm *LeaseManager) AcquireScope(ws WriteScope, holder string, ttl time.Duration) (*WriteScopeLease, error)
AcquireScope takes an exclusive write-scope lease in the DEFAULT cell. It rejects a non-lockable (prose) scope with ErrNotLockable. It rejects a scope that overlaps a live lease held by a different owner with ErrScopeConflict — for ScopePath the overlap test is canonical-prefix containment. A caller re-acquiring a scope it already holds succeeds (idempotent). epoch is the LeadershipEpoch stamped on the issued lease — pass 0 from a non-leader caller; the scheduler passes the epoch it holds so a scope lease issued by a stale queen is fenceable.
func (*LeaseManager) AcquireScopeEpoch ¶
func (lm *LeaseManager) AcquireScopeEpoch(ws WriteScope, holder string, ttl time.Duration, epoch int64) (*WriteScopeLease, error)
AcquireScopeEpoch is AcquireScope with an explicit LeadershipEpoch stamped on the issued WriteScopeLease (§15.1: the queen issues scope leases as part of its fenced serial order), in the DEFAULT cell. A scope lease carrying an old epoch is fenceable by ValidateEpoch.
func (*LeaseManager) AcquireScopeFenced ¶
func (lm *LeaseManager) AcquireScopeFenced(ws WriteScope, holder string, ttl time.Duration, epoch int64, cell string, fence FenceRef) (*WriteScopeLease, error)
AcquireScopeFenced is AcquireScopeInCell whose final WriteScopeLease write is routed through the §15.1 FENCED-WRITE primitive (Round-7 finding #1): the lease object is created/updated ATOMICALLY with the scheduler-lease fence, so a write-scope lease a STALE queen tried to issue fails ErrFenceStale rather than slipping through after the queen lost leadership. The conflict scan itself is still serialised by scopeMu (the single-process MVP boundary lease.go documents); the fence closes the leadership-TOCTOU gap on the lease WRITE — the correctness-bearing step the admission transaction owns.
fence is the scheduler ServiceLease the admitting queen holds. A zero-valued fence means no leadership is asserted (a non-leader / test caller) and the write takes the plain Create/Update path — exactly AcquireScopeInCell.
func (*LeaseManager) AcquireScopeInCell ¶
func (lm *LeaseManager) AcquireScopeInCell(ws WriteScope, holder string, ttl time.Duration, epoch int64, cell string) (*WriteScopeLease, error)
AcquireScopeInCell is AcquireScopeEpoch scoped to a named §15.4 cell (finding #7 — multi-cell isolation). The issued lease's object Name is namespaced by cell, and the conflict scan considers ONLY leases in the same cell, so two cells sharing one Store can each hold an overlapping write scope without a false ErrScopeConflict. An empty cell falls back to the default cell — so AcquireScopeEpoch is exactly AcquireScopeInCell(..., DefaultCell).
func (*LeaseManager) AcquireScopeLeaseByIdentity ¶
func (lm *LeaseManager) AcquireScopeLeaseByIdentity(id WriteScopeLeaseIdentity, ttl time.Duration, epoch int64, fence FenceRef) (*WriteScopeLease, error)
AcquireScopeLeaseByIdentity re-creates or renews the exact acquisition identity recorded on a durable Process. It is intentionally narrower than AcquireScopeFenced: Process liveness checks match on InstanceID, so a scheduler repairing a missing running-lane lease must not mint a fresh instance and leave /proc unable to prove the lane useful.
func (*LeaseManager) AcquireService ¶
func (lm *LeaseManager) AcquireService(service, holder string, ttl time.Duration) (*ServiceLease, error)
AcquireService attempts single-activation leader election for a service. It succeeds if no lease object exists, or the existing lease's TTL has expired, or the caller is already the holder (idempotent re-acquire). It fails with ErrLeaseHeld if a different holder owns a live lease.
On a FRESH take-over — no lease existed, or the existing lease was expired or held by someone else — the returned lease carries a NEW InstanceID and a LeadershipEpoch incremented past whatever the previous lease held (§15.1 fencing: every take-over advances the epoch). On an IDEMPOTENT re-acquire by the current holder, the InstanceID and Epoch are PRESERVED — re-acquiring your own live lease is not a leadership change.
Concurrency safety: when many callers race, every loser hits the store's compare-and-swap (Create returns ErrAlreadyExists, or Update returns *ConflictError) and gets ErrLeaseHeld. Exactly one caller wins.
func (*LeaseManager) AcquireServiceInstance ¶
func (lm *LeaseManager) AcquireServiceInstance(service, holder, instanceID string, ttl time.Duration) (*ServiceLease, error)
AcquireServiceInstance is the same single-activation acquire as AcquireService, but authenticated by a caller-minted per-activation instance token. If a live lease already exists for the same holder but a different InstanceID, this call rejects it with ErrLeaseHeld instead of treating it as an idempotent re-acquire.
func (*LeaseManager) ReleaseScopeInstance ¶
func (lm *LeaseManager) ReleaseScopeInstance(ws WriteScope, holder, instanceID string) error
ReleaseScopeInstance relinquishes a held write-scope lease in the DEFAULT cell, authenticated by the immutable InstanceID minted by AcquireScope.
func (*LeaseManager) ReleaseScopeInstanceInCell ¶
func (lm *LeaseManager) ReleaseScopeInstanceInCell(ws WriteScope, holder, cell, instanceID string) error
ReleaseScopeInstanceInCell relinquishes a held write-scope lease in a named §15.4 cell, authenticated by the immutable acquisition InstanceID. An empty cell falls back to the default cell.
func (*LeaseManager) ReleaseScopeLeaseByIdentity ¶
func (lm *LeaseManager) ReleaseScopeLeaseByIdentity(id WriteScopeLeaseIdentity, writer LaneResourceWriter) (bool, error)
ReleaseScopeLeaseByIdentity releases and deletes exactly one scope-lease acquisition. If the named lease is absent, already released, expired, or now held by a different acquisition, the call is an idempotent no-op. CAS/fence conflicts are returned so the caller retries the whole lane cleanup next pass.
The release is ONE RV-guarded delete of the exact live acquisition. The previous shape — Update(clear holder) then a separate re-read + Delete — could land the update and lose the delete (crash, backend error), leaving a holder-empty row that no retry matched (the identity no longer held) and that admission's OpCreate then tripped over FOREVER: every later admission of the same deterministic scope name failed "grant_already_exists" with no code path deleting the debris. A wedge of the whole write scope.
func (*LeaseManager) ReleaseService ¶
func (lm *LeaseManager) ReleaseService(service, holder string) error
ReleaseService is the legacy holder-only release path. It succeeds only for pre-fencing leases with no InstanceID. Modern per-acquisition leases must use ReleaseServiceInstance so a stale holder string cannot release a successor.
Release is a compare-and-swap, not a blind delete. We CAS the lease into a released (holder-cleared, expired) state on the exact ResourceVersion we validated; if anyone mutated the lease meanwhile the CAS loses with ErrConflict — which means we no longer hold it, so we return ErrNotHolder and never clobber the current holder.
func (*LeaseManager) ReleaseServiceInstance ¶
func (lm *LeaseManager) ReleaseServiceInstance(service, instanceID string) error
ReleaseServiceInstance relinquishes a held service lease, authenticated by the immutable InstanceID minted on acquire. A stale former holder reusing the same holder string cannot release its successor's lease — the InstanceID will not match (audit finding: fence release by lease-instance identity).
func (*LeaseManager) RenewScopeLeaseByIdentity ¶
func (lm *LeaseManager) RenewScopeLeaseByIdentity(id WriteScopeLeaseIdentity, ttl time.Duration, epoch int64, fence FenceRef) (*WriteScopeLease, error)
func (*LeaseManager) RenewService ¶
func (lm *LeaseManager) RenewService(service, holder string) (*ServiceLease, error)
RenewService extends a held service lease, authenticated by the holder STRING. It fails with ErrNotHolder if the caller does not currently hold it (expiry counts as not holding).
FENCING NOTE. RenewService is different-holder-safe: a renew by a string that is not the current holder fails. It is NOT same-holder-safe: a stale process that crashed and was succeeded by a new process REUSING THE SAME HOLDER STRING would pass the string check. For same-holder stale-reuse safety, use RenewServiceInstance, which authenticates the immutable per-acquisition InstanceID. The kernel's own queen (the scheduler) uses the instance-fenced path; RenewService remains for callers whose holder identity is unique per process by construction.
func (*LeaseManager) RenewServiceInstance ¶
func (lm *LeaseManager) RenewServiceInstance(service, instanceID string) (*ServiceLease, error)
RenewServiceInstance extends a held service lease, authenticated by the immutable InstanceID minted on acquire (ServiceLease.Spec.InstanceID). It fails with ErrNotHolder unless the stored lease is live AND carries exactly that InstanceID — so a stale former holder, even one reusing the same holder string after a successor reacquired, cannot renew the successor's lease (audit finding: fence renew/release by lease-instance identity, not holder string).
func (*LeaseManager) ScopeLeaseByIdentity ¶
func (lm *LeaseManager) ScopeLeaseByIdentity(id WriteScopeLeaseIdentity) (*WriteScopeLease, error)
ScopeLeaseByIdentity reads the exact scope-lease object named by (scope, cell). It intentionally does not list all scope leases.
func (*LeaseManager) ServiceLeaseEpoch ¶
func (lm *LeaseManager) ServiceLeaseEpoch(service string) (epoch int64, live bool, err error)
ServiceLeaseEpoch returns the current LeadershipEpoch of a service lease and whether the lease is live. A stale-leader fencing check uses it: the queen holds the epoch from its acquire, and ValidateEpoch compares a write's epoch against the current one. A non-existent lease yields (0, false, nil).
func (*LeaseManager) ServiceLeaseFencing ¶
func (lm *LeaseManager) ServiceLeaseFencing(service string) (LeaseFencing, error)
ServiceLeaseFencing returns the full fencing identity of a service lease — the additive §15.1 counterpart of ServiceLeaseEpoch. The admission transaction re-reads it before every correctness-bearing write so a queen that lost the lease (expired, released, or taken over by a successor that re-used the epoch number) is fenced even when the bare Epoch still matches. A non-existent lease yields a zero LeaseFencing (Live false) and no error.
type ListOptions ¶
ListOptions bounds a Store.ListPage call. Continue is an opaque token returned by a prior ListPage call with the same kind and selector.
type MemJournal ¶
type MemJournal struct {
// contains filtered or unexported fields
}
MemJournal is the in-memory append-only event log with consumer groups and committed cursors. It preserves replay/retention semantics while the process stays alive; production durability is supplied by pkg/etcdstore.EtcdJournal.
func NewJournal ¶
func NewJournal(maxRetained int) *MemJournal
NewJournal returns an empty journal. maxRetained bounds the live log; pass 0 for DefaultJournalRetention. Compaction never trims past the slowest LIVE committed cursor, so a lagging consumer is never starved of its replay; a group abandoned without an Unsubscribe is retired via groupTTL (see NewJournalWithTTL) so a dead group cannot pin retention forever.
func NewJournalWithTTL ¶
func NewJournalWithTTL(maxRetained int, groupTTL time.Duration, clock Clock) *MemJournal
NewJournalWithTTL returns an empty journal with an explicit dead-consumer retirement TTL. groupTTL bounds how long a consumer group may go without a Poll/Commit before compaction stops honouring its cursor: a group crashed without an Unsubscribe would otherwise pin `base` forever and let the live log grow without bound under sustained Append load, so maxRetained would not be a hard bound at all (the §7.7 retention finding). With a TTL an abandoned group is dead for compaction once silent, and maxRetained is a genuine ceiling. groupTTL <= 0 disables TTL retirement (a cursor honoured until an explicit Unsubscribe). clock may be nil for the system clock — an injectable seam so retirement tests do not sleep.
func (*MemJournal) Append ¶
func (j *MemJournal) Append(ev Event) (int64, error)
Append writes an event and returns the assigned Offset. The supplied event's Offset and Timestamp are overwritten by the journal. The event's Payload is deep-copied before it is retained: a caller that mutates its payload map after Append cannot reach into and corrupt journal history — the journal is append-only, a written event immutable. Append wall-clock is folded into the exported AppendStats metric (the §7.7 / §11.2 append-latency instrumentation).
func (*MemJournal) AppendStats ¶
func (j *MemJournal) AppendStats() AppendStats
AppendStats returns the journal's exported Append-latency metric — the §7.7 / §11.2 instrumentation that lets a load harness attribute a super-linear journal sub-phase. It is a measured internal counter, not an external timer.
func (*MemJournal) BeginGroupApply ¶
func (j *MemJournal) BeginGroupApply(group string) (func(), error)
BeginGroupApply marks a consumer group as actively applying a polled batch. Compaction treats an in-flight group as live even when its handler runs past groupTTL, so a slow-but-live consumer cannot lose uncommitted history.
func (*MemJournal) ClearConsumerFailure ¶
func (j *MemJournal) ClearConsumerFailure(group string, sourceOffset int64) error
ClearConsumerFailure drops retry state for a source event that was successfully committed by its consumer group.
func (*MemJournal) Commit ¶
func (j *MemJournal) Commit(group string, cursor int64) error
Commit durably advances the group's cursor. cursor is the Offset the group has fully processed up to and including; the next Poll starts at cursor+1. A cursor behind the current committed position returns ErrCursorRegressed — cursors only move forward.
func (*MemJournal) CommittedCursor ¶
func (j *MemJournal) CommittedCursor(group string) (int64, error)
CommittedCursor returns the group's committed cursor (the Offset of the next un-acked event). It exists for /proc and for backpressure: cursor lag — head minus this — is the verify-stage backpressure signal of §7.7.
func (*MemJournal) EnforceRetention ¶
func (j *MemJournal) EnforceRetention(forceDropHorizon int64) (RetentionResult, error)
EnforceRetention implements RetentionEnforcer for the in-memory journal.
func (*MemJournal) Head ¶
func (j *MemJournal) Head() (int64, error)
Head returns the Offset the next Append will use — i.e. the count of events ever appended.
func (*MemJournal) Lag ¶
func (j *MemJournal) Lag(group string) (int64, error)
Lag returns how many events the group has not yet committed (head minus its cursor). Growing lag is the signal admission throttles against.
func (*MemJournal) Poll ¶
func (j *MemJournal) Poll(group string) ([]Event, error)
Poll returns every event the group has not yet committed, oldest first, without advancing the cursor — the group must Commit to advance. Re-Poll before a Commit returns the same events (at-least-once delivery).
func (*MemJournal) PollN ¶
func (j *MemJournal) PollN(group string, limit int) ([]Event, error)
PollN returns at most limit uncommitted events without advancing the cursor. A non-positive limit is the same as Poll. It is an additive MemJournal helper for batch consumers; the stable Journal interface remains unchanged.
func (*MemJournal) RecordConsumerFailure ¶
func (j *MemJournal) RecordConsumerFailure(group string, source Event, maxAttempts int, cause string) (ConsumerFailureRecord, error)
RecordConsumerFailure advances the shared poison retry budget for (group, source.Offset). Once the budget reaches maxAttempts it appends a single EventDeadLetter while holding the journal lock, making the DLQ record idempotent across replacement and concurrent consumers.
func (*MemJournal) RetentionStats ¶
func (j *MemJournal) RetentionStats() (JournalRetentionStats, error)
RetentionStats returns the in-memory journal's retained-size snapshot.
func (*MemJournal) RetireDeadGroups ¶
func (j *MemJournal) RetireDeadGroups() []string
RetireDeadGroups removes every consumer group silent past groupTTL — explicit, observable dead-group reaping. compactLocked already IGNORES a dead group's cursor, so retention is bounded without this call; this additionally DELETES the dead cursors so a /proc group listing and CommittedCursor stop reporting a zombie. It returns the retired names; a no-TTL journal retires nothing.
A group retired while it still had un-acked events (cursor < head) is an abandonment with potential audit loss: the un-acked range stops being pinned and later compaction may trim it before any revival. That is recorded as an EventDeadLetter audit marker (reason=group_retired_with_lag) BEFORE the delete — never a silent drop (H3). The marker names the un-acked lag as the loss upper bound; a revival that replays from the retained log polls the marker and can reconcile.
func (*MemJournal) Subscribe ¶
func (j *MemJournal) Subscribe(group string) error
Subscribe registers a consumer group. A new group starts at the oldest retained offset (it replays all live history). A group that already exists keeps its committed cursor — re-Subscribe after a crash is a no-op on the cursor, which is exactly the replay-from-last-commit guarantee. Subscribe also refreshes the group's last-activity time, so a crashed-then-recovered consumer that re-Subscribes is immediately live again for retention.
A revived group's cursor is clamped forward to j.base. A group dead past groupTTL no longer pins retention, so compaction can advance `base` past its stale cursor; a re-Subscribe racing RetireDeadGroups would otherwise revive it with a sub-base cursor that compaction honours as a live pin and trims below `base` (a negative `drop`, the maxRetained bound broken) — the clamp anchors the revival to base, the same replay Poll enforces on its own path. The clamp skips events the group never acked, so it is recorded as an EventDeadLetter audit marker (reason=cursor_reanchor_compaction) — never a silent drop (H3).
func (*MemJournal) Unsubscribe ¶
func (j *MemJournal) Unsubscribe(group string) error
Unsubscribe retires a consumer group: its cursor is removed, so compaction no longer pins retention to it. It is the EXPLICIT dead-group retirement path — a cleanly shutting-down consumer calls it so its cursor stops holding the live log (the §7.7 "maxRetained is a hard bound" guarantee). Unsubscribe of an unknown group is a no-op. After it, compaction may trim past the cursor.
type MemStore ¶
type MemStore struct {
// contains filtered or unexported fields
}
MemStore is the in-memory Store backend. It is correct and concurrency-safe: every read returns an independent copy, every write is CAS-checked under a single lock, and Watch fan-out never blocks a writer (a slow watcher's buffer overflowing closes that watcher only). Phase 1 tests run entirely against MemStore — zero external services.
func (*MemStore) AcquireScopeLease ¶
func (s *MemStore) AcquireScopeLease(lease *WriteScopeLease, cell string, fence *FenceRef) (Object, error)
AcquireScopeLease implements ScopeConflictGuard for MemStore. The indexed conflict probe and write both run under s.mu, so two LeaseManagers sharing one MemStore cannot both pass the overlap check before either write lands.
func (*MemStore) Create ¶
Create implements Store. It stamps UID/ResourceVersion and sets Generation to 1 for the new object, then publishes a WatchAdded event.
func (*MemStore) Delete ¶
Delete implements Store. It is a BLIND delete (no ResourceVersion check) — see the Store.Delete contract. It publishes a WatchDeleted event carrying the object's last state.
func (*MemStore) DeleteVersioned ¶
DeleteVersioned is the version-guarded delete: it removes the object only if the stored ResourceVersion equals expectVersion, exactly like Update's compare-and-swap. A stale caller — one whose expectVersion is behind a concurrent update — loses with a *ConflictError and the object is NOT deleted, so a blind delete can never erase a newer object. ErrNotFound is returned for an absent object.
It is the atomic, MemStore-native delete CAS — the whole check-and-delete happens under s.mu. The interface-level DeleteCAS helper routes here when the backend is a *MemStore and falls back to a best-effort Get+Delete otherwise.
func (*MemStore) FencedCreate ¶
FencedCreate implements FencedWriter: it creates obj only if fence's object still holds its expected ResourceVersion/Epoch, with the fence check and the create in ONE critical section under s.mu. A stale fence yields ErrFenceStale and nothing is written; otherwise it behaves exactly like Create (ErrAlreadyExists on a duplicate (Kind, Name)). The fence is checked FIRST so a stale leader is reported as ErrFenceStale even when the create would also have failed for a duplicate.
func (*MemStore) FencedUpdate ¶
FencedUpdate implements FencedWriter: it compare-and-swaps obj only if fence's object still holds its expected ResourceVersion/Epoch, with the fence check and the CAS in ONE critical section under s.mu. A stale fence yields ErrFenceStale and nothing is written; otherwise it behaves exactly like Update — a *ConflictError on a stale obj ResourceVersion, ErrNotFound on an absent obj. The fence is checked FIRST.
func (*MemStore) List ¶
List implements Store. Results are sorted by Name for deterministic iteration (the scheduler needs a stable order before applying priority).
func (*MemStore) ListPage ¶
ListPage implements Store. It bounds object copies before returning a page; Limit <= 0 preserves full List behaviour with an empty continuation.
func (*MemStore) LiveScopeLeasesByHolder ¶
func (s *MemStore) LiveScopeLeasesByHolder(cell, holder string) ([]*WriteScopeLease, error)
func (*MemStore) ReleaseScopeLease ¶
func (s *MemStore) ReleaseScopeLease(leaseName string, scope WriteScope, holder, cell, instanceID string) error
ReleaseScopeLease implements ScopeConflictGuard release for MemStore under the same store mutex. instanceID is required and must match the exact acquisition being released.
func (*MemStore) RenewScopeLeaseIdentity ¶
func (*MemStore) SetClock ¶
SetClock injects the wall-clock MemStore-backed lease helpers read. It exists so a test can advance time past a lease TTL deterministically without sleeping — the same Clock seam lease.go uses. A nil clock restores the system clock. It takes s.mu so it is data-race-safe with concurrent store operations, though in practice a test sets it once before exercising the store.
func (*MemStore) Stats ¶
func (s *MemStore) Stats() StoreStats
Stats returns the MemStore's internal counters — the §11.2 instrumentation surface. CASConflicts is the store's own count of compare-and-swap losers, so a load harness can attribute write contention to the store rather than inferring it from its own retry loops.
func (*MemStore) Update ¶
Update implements Store. It is the compare-and-swap: the supplied object's ResourceVersion must equal the stored version. On success the stored ResourceVersion is bumped and a WatchModified event is published; on a stale version a *ConflictError is returned and nothing is mutated.
func (*MemStore) Watch ¶
func (s *MemStore) Watch(kind Kind, stop <-chan struct{}) (<-chan WatchEvent, error)
Watch implements Store. The returned channel first replays current state as WatchAdded events, then streams live changes until stop is closed. The channel is buffered to hold the full initial snapshot plus live-event slack, so the replay never blocks a kernel writer under s.mu. A watcher whose buffer later overflows on live traffic is dropped (its channel closed) rather than applying backpressure — a slow consumer must never stall the kernel.
func (*MemStore) WatchFrom ¶
func (s *MemStore) WatchFrom(kind Kind, snapshotRev int64, stop <-chan struct{}) (<-chan WatchEvent, error)
WatchFrom implements RevisionWatcher for MemStore. MemStore keeps no history, so it can attach only if snapshotRev is still the current store revision. A caller that sees ErrConflict must retry its bounded snapshot before emitting.
func (*MemStore) WatchSnapshot ¶
func (s *MemStore) WatchSnapshot(kind Kind, sel Selector, limit int, stop <-chan struct{}) (ListPage, <-chan WatchEvent, error)
WatchSnapshot implements SnapshotWatcher for MemStore. The bounded snapshot and live watcher registration happen under one s.mu acquisition, so there is no snapshot-to-attach gap where a write can advance the store revision and force a retry loop. If the matching set is over limit, no watcher is registered and the returned page has NextContinue set.
type MetricGauge ¶
type MetricGauge struct {
Name string `json:"name"`
Value float64 `json:"value"`
Unit string `json:"unit,omitempty"`
}
MetricGauge is one named numeric gauge a non-LLM worker (a trainer, a data pipeline) reported — the durable last reading of e.g. loss / flaw_f1 / fights-per-min. Name is opaque (the kernel branches on no metric name); Unit is a free-form display hint. This is the terminal-surviving twin of the live Rollup metric map: without it a finished training run loses its final loss/F1 the moment the in-memory ring evicts.
type Node ¶
type Node struct {
ObjectMeta `json:",inline"`
Spec NodeSpec `json:"spec"`
Status NodeStatus `json:"status"`
}
Node is a machine. It reports measured capacity and hosts the warm workspace pool.
func (*Node) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the Node.
type NodeLifecycle ¶
type NodeLifecycle string
NodeLifecycle names the lifecycle class of a Node. Empty is valid and means unspecified; placement logic can interpret it later.
const ( NodeLifecycleOnDemand NodeLifecycle = "on-demand" NodeLifecycleSpot NodeLifecycle = "spot" NodeLifecyclePreemptible NodeLifecycle = "preemptible" )
type NodeSpec ¶
type NodeSpec struct {
Hostname string `json:"hostname"`
Zone string `json:"zone,omitempty"`
Region string `json:"region,omitempty"`
Cloud CloudProvider `json:"cloud,omitempty"`
InstanceType string `json:"instanceType,omitempty"`
Lifecycle NodeLifecycle `json:"lifecycle,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Capabilities map[string]string `json:"capabilities,omitempty"`
}
NodeSpec is the desired-state of a Node.
type NodeStatus ¶
type NodeStatus struct {
Status `json:",inline"`
Offers []ResourceOffer `json:"offers,omitempty"`
}
NodeStatus extends Status with the node's measured offers.
type Object ¶
type Object interface {
// GetMeta returns a pointer to the embedded envelope so the store can
// stamp UID/ResourceVersion/Generation in place.
GetMeta() *ObjectMeta
// DeepCopyObject returns an independent copy. The store hands out copies
// so a caller mutating a result cannot corrupt stored state.
DeepCopyObject() Object
}
Object is implemented by every durable kernel kind. The store works against this interface; concrete kinds embed ObjectMeta to satisfy most of it.
func FencedCreate ¶
FencedCreate creates obj through the backend's atomic fenced-write path: obj is created only if fence's object still satisfies the FenceRef at the instant the create commits (§15.1, finding #1). It is the kernel-level entry point the admission transaction and the Spawner route their correctness-bearing creates through.
It FAILS CLOSED — ErrFenceUnsupported — on a Store that does not implement FencedWriter: a fenced write whose atomicity the backend cannot guarantee must not degrade to a plain Create. An invalid FenceRef (no Kind/Name) is a caller bug and is rejected before any store call.
func FencedUpdate ¶
FencedUpdate compare-and-swaps obj through the backend's atomic fenced-write path: obj is updated only if fence's object still satisfies the FenceRef at the instant the update commits (§15.1, finding #1). It is the kernel-level entry point for a fenced CAS — the admission transaction's ticket-status transition routes through it.
Like FencedCreate it FAILS CLOSED (ErrFenceUnsupported) on a backend without FencedWriter and rejects an invalid FenceRef.
func NewObject ¶
NewObject returns a fresh, zero-valued Object of the given Kind, with its ObjectMeta.Kind already stamped. It is the kernel's Kind→zero-object constructor: a serialising backend calls it to obtain the typed value to JSON-unmarshal a stored blob into, so every backend shares one codec instead of re-implementing the Kind dispatch.
NewObject covers the durable kinds declared in object.go AND every extension kind registered through RegisterExtensionKind, so a backend that resolves constructors through NewObject round-trips both the kernel's own kinds and the FC verify/provider/driver kinds. An unregistered Kind returns ErrUnknownKind, never a nil object.
The constructed object is nil-guarded before GetMeta() is dereferenced. RegisterExtensionKind already rejects a nil-producing constructor at registration, and the kernel's own kindRegistry constructors are correct by construction; the guard here is defence-in-depth so a registry that was somehow corrupted surfaces ErrBadConstructor instead of a nil dereference panic that takes down the kernel.
type ObjectMeta ¶
type ObjectMeta struct {
Kind Kind `json:"kind"`
Name string `json:"name"`
UID string `json:"uid"`
Generation int64 `json:"generation"`
ResourceVersion int64 `json:"resourceVersion"`
OwnerRefs []OwnerRef `json:"ownerRefs,omitempty"`
Finalizers []string `json:"finalizers,omitempty"`
CreatedAt time.Time `json:"createdAt"`
// Labels are an indexable string map; Store.List selects against them.
Labels map[string]string `json:"labels,omitempty"`
// LeadershipEpoch is the §15.1/§15.5 fencing token. The leader-elected
// scheduler (the queen) stamps the epoch it held when it authored a
// correctness-bearing object (a Process, a ResourceGrant, a WriteScopeLease
// it issued, the ServiceLease it took). A monotonically increasing epoch is
// minted on every leadership takeover; a stale former leader that resumes
// after its lease expired carries an old epoch and is rejected everywhere
// (FenceEpoch). Zero means "no epoch asserted" — set by non-leader writers
// and by callers that predate fencing; it never fences a write out, so the
// field is purely additive.
LeadershipEpoch int64 `json:"leadershipEpoch,omitempty"`
}
ObjectMeta is the Kubernetes-style envelope every durable kernel object carries. Generation increments on every spec write; ResourceVersion increments on every store write (spec or status) and is the compare-and-swap token used by Store.Update.
func CloneMeta ¶
func CloneMeta(m ObjectMeta) ObjectMeta
CloneMeta returns an independent deep copy of an ObjectMeta — every nested slice (OwnerRefs, Finalizers) and the Labels map is freshly allocated, so a caller mutating the copy cannot corrupt the original. It is the kernel's own envelope-clone helper, exported so a package that embeds ObjectMeta in a new kind (the FC verification, provider, and driver kinds) implements DeepCopyObject by calling one kernel-owned function rather than each re-deriving the field-by-field clone. The kernel's own kinds route their unexported clone() through it, so there is exactly one envelope-copy definition (the deep-copy-at-every-boundary invariant, §6).
func (*ObjectMeta) GetMeta ¶
func (m *ObjectMeta) GetMeta() *ObjectMeta
GetMeta satisfies Object for any kind that embeds ObjectMeta.
type OfferSource ¶
type OfferSource interface {
CurrentOffers() []ResourceOffer
}
OfferSource is the node-agent seam: it returns the currently measured resource offers. The scheduler consumes offers; it never measures the host (§7.5 two-level scheduling).
type OpType ¶
type OpType string
OpType names one write in a Store.Txn operation list.
const ( // OpPut creates or replaces Object. Callers usually pair updates with a // CondResourceVersion condition on the same object. OpPut OpType = "put" // OpCreate creates Object only if it is absent. OpCreate OpType = "create" // OpDelete deletes Kind/Name only if it is present. OpDelete OpType = "delete" )
type OwnerRef ¶
OwnerRef is a typed back-reference to an owning object, used for cascading delete and grant ownership (a ResourceGrant is owned by its Process).
type PhaseStat ¶
type PhaseStat struct {
Phase TickPhase `json:"phase"`
Count int64 `json:"count"`
TotalLatency time.Duration `json:"totalLatency"`
MeanLatency time.Duration `json:"meanLatency"`
MaxLatency time.Duration `json:"maxLatency"`
}
PhaseStat is a snapshot of one tick sub-phase's latency.
type PoisonLaneReason ¶
type PoisonLaneReason string
PoisonLaneReason classifies why a lane was quarantined. It is recorded on the terminal Lane condition so an operator can grep the cause in /proc and audit.
const ( // PoisonMissingGrant: a still-non-terminal lane's Process references a // ResourceGrant that no longer exists in the store. The running-source // rebuild cannot account the lane (a missing grant is a torn-state risk), // and the lane can never make progress because its grant is gone. Before // this quarantine path existed, this returned a hard error from // RunningLanes() that failed the WHOLE scheduler tick — five such ticks in // a row escalated the scheduler past its restart budget and HALTED the // entire control plane for launchd relaunch (the 2026-05-31 crash loop: // `process ... references missing grant ... (lane ... still RUNNING): object // not found`). One poison lane must never kill scheduling for every other // lane. PoisonMissingGrant PoisonLaneReason = "missing_grant" // PoisonScopeUnreconcilable: a still-live lane's running write-scope lease // could not be renewed or re-acquired for a reason that is neither a // transient CAS conflict (retried with backoff) nor a benign // not-found/not-holder (re-acquired). The lane's scope exclusion cannot be // reasserted, so it is quarantined rather than failing the tick. PoisonScopeUnreconcilable PoisonLaneReason = "scope_unreconcilable" )
type Preemption ¶
type Preemption struct {
Victim RunningLane
OnBehalfOf Ticket
ReleasedAmounts ResourceRequest
// Fence is the scheduler ServiceLease fence that must hold atomically with
// the victim phase write. It closes the stale-leader gap between the
// admission transaction's pre-write leadership recheck and a provider-owned
// preemptor's own Store.Txn.
Fence FenceRef
}
Preemption is a scheduler decision to free a lower-priority RUNNING lane so a higher-priority ticket that did not otherwise fit can be admitted (the §7.2 Borg priority+preemption contract). It is the kernel-owned preemption surface the loop turns into a LanePreempted transition + a journaled event.
Victim is the running lane chosen for preemption — the lowest-priority preemptible lane whose released resources help the blocked ticket. OnBehalfOf is the higher-priority ticket the preemption serves. ReleasedAmounts is the per-currency reservation freed by stopping the victim (the sum of its grants), which the scheduler returns to the ledger before re-admitting.
type Preemptor ¶
type Preemptor interface {
PreemptLane(p Preemption) error
}
Preemptor is the kernel side-effect the scheduler loop invokes for each preemption decision: it drives the victim lane's PREEMPTED transition (its durable execution resumes from the journal later — §7.6) and releases the victim's resources. It is the §7.2 preemption seam, the counterpart of Spawner. A scheduler wired without a Preemptor still PRODUCES Preemption decisions (visible in TickResult and journaled) but does not enact them.
type Process ¶
type Process struct {
ObjectMeta `json:",inline"`
Spec ProcessSpec `json:"spec"`
Status Status `json:"status"`
}
Process is an OS process the kernel admitted. It is authoritative: a process the kernel did not record is an orphan and is reaped within a tick.
func (*Process) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the Process.
type ProcessSpec ¶
type ProcessSpec struct {
ParentUID string `json:"parentUID,omitempty"`
PID int `json:"pid"`
Cgroup string `json:"cgroup,omitempty"`
LaneName string `json:"laneName,omitempty"`
// TicketName is the typed owner of the lane/process. LaneName is an
// execution identity; TicketName is the work identity controllers use for
// distributed absence/adoption checks without parsing lane-name strings.
TicketName string `json:"ticketName,omitempty"`
// GrantNames are the ResourceGrant objects this process owns; releasing
// the process releases the grants.
GrantNames []string `json:"grantNames,omitempty"`
// ScopeLeases are the exact WriteScopeLease acquisitions this process owns.
// They make cleanup independent of the Ticket object: a deleted ticket no
// longer removes the only durable record of which scope leases to release.
ScopeLeases []WriteScopeLeaseIdentity `json:"scopeLeases,omitempty"`
}
ProcessSpec is the desired-state of a Process.
type ProcessStatusExt ¶
type ProcessStatusExt struct {
LastHeartbeat time.Time `json:"lastHeartbeat"`
RSSBytes int64 `json:"rssBytes"`
CPUSample float64 `json:"cpuSample"`
}
ProcessStatusExt holds the high-churn liveness fields; the durable Process object carries only the last-known values, and the journal carries the stream. LastHeartbeat is the measured-truth liveness signal.
type ProgressGauge ¶
type ProgressGauge struct {
Current int64 `json:"current"`
Total int64 `json:"total,omitempty"`
Unit string `json:"unit,omitempty"`
}
ProgressGauge is the durable last "how far along" fraction (Current of Total units, e.g. step 357 of 2000). Total <= 0 means the worker reported a position with no known bound. The kernel never interprets the unit.
type ReconcileInput ¶
type ReconcileInput struct {
// Runnable is the set of triage tickets considered for admission this tick.
Runnable []Ticket
// Running is the set of already-running lanes. Their reservations are
// folded back into the ledger (cross-tick accounting) and they are the
// candidate victims for preemption.
Running []RunningLane
// Ledger is the measured capacity from the node-agent's offers. It is
// CLONED inside ReconcileColony — the caller's ledger is never mutated.
Ledger *ResourceLedger
// Epoch is the LeadershipEpoch the deciding queen holds. It is stamped onto
// every ResourceGrant the decision issues so a grant a stale queen issued
// is fenceable, and it makes issued grant names collision-free across
// leader handoffs.
Epoch int64
// AdmissionGate is the OPTIONAL pre-admission hook (scheduler_admission_gate.go).
// When non-nil, its pure HoldReason predicate partitions held tickets out of
// admission BEFORE the resource/scope/preemption pipeline. Nil is fail-open —
// the decision is byte-identical to the no-gate path.
AdmissionGate AdmissionGate
// ExclusiveContended is this tick's exclusive-currency contention latch
// (exclusive_contention.go): currencies with fresh evidence of an
// out-of-band holder (a backpressure exit stamped on a ticket that
// requested them). A runnable ticket requesting a latched currency is held
// in the typed WaitingExclusiveContended wait — no lane attempt, no
// workspace claim, no preemption — instead of admitting into a launch-time
// refusal. Nil/empty is fail-open: the decision is identical to the
// no-latch path.
ExclusiveContended map[Currency]bool
}
ReconcileInput is the full input to the §15 colony decision core. It carries everything ReconcileColony needs to make a cross-tick, preemption-aware, epoch-fenced decision in one pure pass.
type RefillSource ¶
RefillSource returns additional runnable tickets when the primary runnable source is short of the configured lane floor.
SLICE 1 — MECHANISM ONLY (dormant by default). The floor is plumbed through the kernel's internal lane budget, but the PUBLIC policy.LaneTargetPolicySpec / proto do not yet expose MinLaneFloor and no production RefillSource is wired — so with the seeded default (floor 0 / Refill nil) this path is INERT and is exercised only by tests. Slice 2 (activation) adds the public policy field + proto + a concrete RefillSource (bounded redrive of escalated work / roadmap-derived next work) and sets a non-zero floor; only then does this change org behavior.
type ResourceGrant ¶
type ResourceGrant struct {
ObjectMeta `json:",inline"`
Spec ResourceGrantSpec `json:"spec"`
Status Status `json:"status"`
}
ResourceGrant is a reservation issued by admission, owned by a Process, and released when that Process exits.
func (*ResourceGrant) DeepCopyObject ¶
func (g *ResourceGrant) DeepCopyObject() Object
DeepCopyObject returns an independent copy of the ResourceGrant.
func (*ResourceGrant) GetMeta ¶
func (g *ResourceGrant) GetMeta() *ObjectMeta
GetMeta makes ResourceGrant an Object.
type ResourceGrantSpec ¶
type ResourceGrantSpec struct {
TicketName string `json:"ticketName"`
LaneName string `json:"laneName,omitempty"`
ProcessName string `json:"processName,omitempty"`
Amounts ResourceRequest `json:"amounts"`
}
ResourceGrantSpec is the desired-state of a ResourceGrant: the amount granted per currency and the ticket/process it backs.
type ResourceLedger ¶
type ResourceLedger struct {
// contains filtered or unexported fields
}
ResourceLedger is the kernel's single capacity model. Per currency it tracks Capacity, Reservable (= Capacity × (1 − reserve)), Reserved (Σ admitted requests), and Used (measured). Admission gates against every currency at once; the scheduler holds one ledger per reconcile tick.
func LedgerFromOffers ¶
func LedgerFromOffers(offers []ResourceOffer) *ResourceLedger
LedgerFromOffers builds a ledger from the node-agent's measured offers. Each offer's Reservable is derived as Capacity × (1 − Reserve). Reserve is clamped to [0,1]. Offers for the same currency sum (multi-node future).
Capacity and Used are sanitized at ingress: a NaN, an Inf, or a negative measurement is coerced to zero — a misbehaving node-agent offer must never poison the ledger arithmetic that admission depends on.
A Reserve >= 1 means the offer withholds ALL of its capacity — Reservable becomes 0 and the currency admits nothing. The previous code clamped such an offer to 0.99, which FABRICATED 1% of reservable headroom out of a fully withheld or malformed offer: a node/provider that should be unavailable would still admit work, breaking effective-target honesty. The clamp is now 1.0 — a fully reserved offer is honestly zero reservable, never 0.99.
func (*ResourceLedger) ApplyReclamation ¶
func (l *ResourceLedger) ApplyReclamation(overuse ResourceRequest)
ApplyReclamation folds a process's MEASURED OVERUSE into the ledger — the Borg reclamation step (§7.3). For a currency where a running process's measured `used` exceeds what it `reserved`, the kernel re-charges the difference so admission tightens immediately against the real footprint. The caller passes the per-currency overuse delta `max(0, used − reserved)`; ApplyReclamation adds each positive delta to Reserved (and lifts Used to keep Used ≥ Reserved). It is ApplyReservation's measured-overuse companion and is applied per running process during the tick's ledger rebuild.
func (*ResourceLedger) ApplyReservation ¶
func (l *ResourceLedger) ApplyReservation(amounts ResourceRequest)
ApplyReservation adds an already-granted reservation to the ledger's Reserved totals WITHOUT the admission gate — it is not a new admission, it is the kernel re-stating a reservation that durable state says already exists. The scheduler calls it once per outstanding ResourceGrant when it rebuilds the ledger at the start of a tick (and on leader handoff), so an already-running lane correctly reduces the next tick's headroom. A non-finite or negative amount is skipped (defence-in-depth, the same as Reserve).
This is the missing half of cross-tick admission. Reserve is gated and is for a NEW admission this tick; ApplyReservation is ungated and is for a PRIOR admission carried forward. Without it, Reserved starts at 0 every tick and effective_target can exceed real admissible capacity — the v1-class over-admission bug.
func (*ResourceLedger) Available ¶
func (l *ResourceLedger) Available(c Currency) float64
Available returns the unreserved headroom for a currency.
func (*ResourceLedger) CanReserve ¶
func (l *ResourceLedger) CanReserve(req ResourceRequest) bool
CanReserve reports, for every currency, whether req fits in the remaining headroom: req[c] ≤ Reservable[c] − Reserved[c]. It is the predicate admission.TryAdmit evaluates; it never mutates the ledger.
func (*ResourceLedger) Capacity ¶
func (l *ResourceLedger) Capacity(c Currency) float64
Capacity returns the measured total for a currency.
func (*ResourceLedger) Clone ¶
func (l *ResourceLedger) Clone() *ResourceLedger
Clone returns a fully independent copy of the ledger — a fresh map with a fresh currencyLedger per currency. It is what keeps Reconcile a genuinely PURE function: Reconcile clones the caller's ledger and charges the clone, so calling Reconcile twice with the same ledger pointer yields the same result (the previous code charged the caller-owned ledger in place, making repeated calls history-dependent — the §7.2 "pure decision core" was not actually pure). A nil ledger clones to a fresh empty ledger.
func (*ResourceLedger) RechargeUsage ¶
func (l *ResourceLedger) RechargeUsage(delta ResourceRequest)
RechargeUsage implements Borg-style reclamation (§7.3). For a process whose measured usage exceeds its reservation, the kernel re-charges it at measured usage so admission tightens immediately. delta is measuredUsed − reserved per currency; a positive delta is added to Reserved. A negative delta (process under-using) is ignored here — reclaiming under-use into batch headroom is a separate scheduler decision, not a ledger mutation.
func (*ResourceLedger) Release ¶
func (l *ResourceLedger) Release(req ResourceRequest)
Release returns a previously reserved request to the ledger — called when a Process exits and its ResourceGrant is freed. Reserved is floored at 0. A non-finite or negative amount is skipped so a poisoned grant cannot drive Reserved to NaN on release.
func (*ResourceLedger) Reservable ¶
func (l *ResourceLedger) Reservable(c Currency) float64
Reservable returns the admittable ceiling for a currency.
func (*ResourceLedger) Reserve ¶
func (l *ResourceLedger) Reserve(req ResourceRequest) error
Reserve charges req against the ledger. It returns an error (and mutates nothing) if the request does not fit every currency — Reserve is all-or- nothing, the same gate as CanReserve. The scheduler calls Reserve only after TryAdmit reported OK, but Reserve re-checks so a direct caller is safe.
Shortfalls already rejects a request carrying a non-finite or negative amount, so the charge loop below only ever sees usable quantities; the finite guard is defence-in-depth so no NaN can reach Reserved.
func (*ResourceLedger) Reserved ¶
func (l *ResourceLedger) Reserved(c Currency) float64
Reserved returns the sum of admitted requests for a currency.
func (*ResourceLedger) Shortfalls ¶
func (l *ResourceLedger) Shortfalls(req ResourceRequest) []Currency
Shortfalls returns the currencies for which req does not fit, in canonical AllCurrencies order — the typed BlockedOn list admission surfaces in /proc. An empty slice means the request is admissible.
A currency whose requested amount is not a usable quantity (NaN, ±Inf, or negative) is reported as a shortfall: an invalid request can never be admitted, and reporting it (rather than silently skipping it) keeps a poisoned amount from slipping past the gate and corrupting Reserved.
func (*ResourceLedger) Snapshot ¶
func (l *ResourceLedger) Snapshot() map[Currency]ResourceLine
Snapshot returns a flat per-currency view for /proc and tests.
func (*ResourceLedger) Used ¶
func (l *ResourceLedger) Used(c Currency) float64
Used returns the measured live consumption for a currency.
type ResourceLine ¶
type ResourceLine struct {
Capacity float64 `json:"capacity"`
Reservable float64 `json:"reservable"`
Reserved float64 `json:"reserved"`
Used float64 `json:"used"`
Available float64 `json:"available"`
}
ResourceLine is the exported, copyable per-currency ledger view.
type ResourceOffer ¶
type ResourceOffer struct {
Currency Currency `json:"currency"`
// Capacity is the measured total for this currency on the node.
Capacity float64 `json:"capacity"`
// Used is the measured live consumption (Σ worker RSS, slots held, ...).
Used float64 `json:"used"`
// Reserve is the fraction [0,1) held back as headroom; Reservable is
// Capacity × (1 − Reserve).
Reserve float64 `json:"reserve"`
}
ResourceOffer is one currency's capacity as measured and published by a node-agent (§7.5 two-level scheduling). The scheduler consumes offers; it never measures the host itself.
type ResourceRequest ¶
ResourceRequest is an amount per currency — what a Ticket asks for, what a ResourceGrant reserves. A missing currency means zero. Amounts are float64 so ram_bytes and fractional cpu share one type; callers keep ram_bytes integral by convention.
func ProcessMeasuredUsage ¶
func ProcessMeasuredUsage(p Process) ResourceRequest
ProcessMeasuredUsage extracts a Process's last measured per-currency usage from its MeasuredUsageConditionType Condition. A Process that has never been measured (no such Condition, or an empty/garbled Message) yields a nil request — accounted at its reservation only, the safe default. It is the read counterpart of SetProcessMeasuredUsage; the scheduler's running-lane builder calls it so reclamation reads one kernel-owned definition.
func (ResourceRequest) FractionalIntegralCurrencies ¶
func (r ResourceRequest) FractionalIntegralCurrencies() []Currency
FractionalIntegralCurrencies returns the request's integral currencies whose requested amount is a positive non-whole number, in canonical AllCurrencies order followed by any out-of-canon currencies. An empty slice means every integral currency is requested as a whole number. It deliberately ignores finiteness/negativity (the existing negative-only checks at those boundaries already cover those) and zero/absent amounts.
func (ResourceRequest) Get ¶
func (r ResourceRequest) Get(c Currency) float64
Get returns the amount for a currency, or 0 if absent.
func (ResourceRequest) IsEmpty ¶
func (r ResourceRequest) IsEmpty() bool
IsEmpty reports whether the request asks for nothing — every amount is a finite value <= 0. A non-finite amount (NaN, ±Inf) is NOT empty: it is an invalid claim that must be examined by admission, never silently treated as "asks for nothing" (which would let a poisoned request bypass the gate).
type RetentionEnforcer ¶
type RetentionEnforcer interface {
// EnforceRetention force-advances any live consumer group lagging by more
// than forceDropHorizon to the retention floor, recording each forced drop
// as an EventDeadLetter audit marker, then compacts. A non-positive horizon
// disables forced drops (the pass only reports size + compacts), so a
// deployment can opt out and rely on maxRetained + groupTTL alone.
EnforceRetention(forceDropHorizon int64) (RetentionResult, error)
}
RetentionEnforcer is the optional Journal extension the maintenance controller uses to bound retention under a lagging consumer. Both MemJournal and etcdstore.EtcdJournal implement it; a Journal that does not is simply not force-bounded (its compaction still respects maxRetained for fully-silent groups via groupTTL).
type RetentionResult ¶
type RetentionResult struct {
// ForcedDrops is one record per group force-advanced this pass.
ForcedDrops []ForcedDropRecord `json:"forcedDrops,omitempty"`
// Head is the journal head after the pass.
Head int64 `json:"head"`
// Base is the oldest retained offset after the pass.
Base int64 `json:"base"`
// LiveLen is the retained live-log length after the pass (Head - Base for
// the etcd journal; len(log) for the in-memory journal).
LiveLen int64 `json:"liveLen"`
}
RetentionResult is the measured outcome of one EnforceRetention pass: the forced drops it performed and the journal's post-pass size, for /proc.
type RetryBudgetPolicy ¶
type RetryBudgetPolicy struct {
MaxAttempts int
MaxElapsed time.Duration
InitialBackoff time.Duration
MaxBackoff time.Duration
}
RetryBudgetPolicy is the shared bounded-retry policy: attempts and total age cap retry lifetimes; exponential backoff spaces retryable work.
func DefaultLaneRetryPolicy ¶
func DefaultLaneRetryPolicy() RetryBudgetPolicy
DefaultLaneRetryPolicy is the durable activity retry policy used by lane execution and the scheduler's retryable-lane re-drive gate.
func (RetryBudgetPolicy) Advance ¶
func (p RetryBudgetPolicy) Advance(prev DurableRetryState, now time.Time, reason string) (DurableRetryState, bool)
Advance returns the next durable retry state and whether the policy budget is exhausted at now.
type ReversiblePreemptor ¶
type ReversiblePreemptor interface {
Preemptor
// RestorePreemption reverses a previously-enacted PreemptLane: it returns
// the victim lane from PREEMPTED back to RUNNING. It is called by the
// admission transaction's rollback when a step AFTER the preemption (the
// spawn) failed.
RestorePreemption(p Preemption) error
}
ReversiblePreemptor is the OPTIONAL §15.1 transaction-safety extension of Preemptor (finding #2). The admission transaction enacts a ticket's preemptions immediately before the final spawn; if that spawn then fails, the transaction rolls back — and a Preemptor that also satisfies ReversiblePreemptor has every enacted preemption RESTORED (the victim lane returned to RUNNING), so a victim is never left preempted without an admitted replacement.
It is an additive interface: a Preemptor that does NOT implement it still works — the rollback falls back to the documented best-effort, where the next reconcile re-observes the victim's durable state and re-admits it. Adding an interface is permitted by the frozen-ABI rule (it renames/removes nothing).
type RevisionWatcher ¶
type RevisionWatcher interface {
// WatchFrom streams live changes after snapshotRev. It does not replay the
// current state; callers that need a replay must get it from ListPage.
WatchFrom(kind Kind, snapshotRev int64, stop <-chan struct{}) (<-chan WatchEvent, error)
}
RevisionWatcher is the additive watch extension for callers that already hold a bounded snapshot revision and need live changes strictly after it.
type RunnableSource ¶
RunnableSource yields the tickets the scheduler should consider this tick. It must be an indexed read (never a full store scan, §7.2).
func NewIndexedRunnableSource ¶
func NewIndexedRunnableSource(store Store) RunnableSource
NewIndexedRunnableSource returns the kernel's standard RunnableSource over a Store: an indexed List on the triage label plus a bounded dependency-edge check. It is what agentosd wires the scheduler against.
func NewIndexedRunnableSourceForCell ¶
func NewIndexedRunnableSourceForCell(store Store, cell string) RunnableSource
NewIndexedRunnableSourceForCell returns the kernel's standard RunnableSource scoped to a §15.4 cell. A named-cell source selects only triage tickets whose CellLabel is that cell. The DefaultCell source surfaces tickets with no CellLabel (unassigned work for the default/global placer) plus tickets explicitly labelled default. Unassigned tickets are therefore never claimed by arbitrary named-cell queens.
func NewIndexedRunnableSourceForCellWithGlobal ¶
func NewIndexedRunnableSourceForCellWithGlobal(store Store, global Store, cell string) RunnableSource
NewIndexedRunnableSourceForCellWithGlobal returns a cell-scoped RunnableSource for migrated named-cell tickets. Before surfacing a cell-local copy it reads the global placement marker and verifies that the marker still designates the same cell and the same global ticket identity.
type RunningLane ¶
type RunningLane struct {
Ticket Ticket
Process Process
Grants []ResourceGrant
// ScopeLeases are the exact write-scope acquisitions the running lane owns.
// They are copied from Process.Spec.ScopeLeases, or from the Lane row when
// the Process has not yet copied them. Ticket.Spec.WriteScope is only a
// legacy fallback because tickets are mutable and may be absent after bind.
ScopeLeases []WriteScopeLeaseIdentity
// Used is the lane's measured per-currency consumption (Σ worker RSS, the
// CPU sample, slots held). It is the Borg-reclamation input (§7.3): when a
// lane's measured Used exceeds what its grants Reserved, ReconcileColony
// folds the overage `max(0, used−reserved)` into the ledger so admission
// tightens against the real footprint. A nil/empty Used means the lane is
// accounted at its reservation only — the safe, pre-measurement default.
Used ResourceRequest
// NonPreemptible means the lane is in the scheduler view only because it
// still owns resources, not because another agent attempt may be stopped to
// admit higher-priority work. Terminal lanes with retained provider/resource
// grants are charged this way until cleanup proves the process is absent.
NonPreemptible bool
// PreemptionBlocked means the lane is still executing and must keep normal
// accounting/scope renewal, but an embedding-owned runtime guard says it is
// not currently a safe preemption victim.
PreemptionBlocked bool
// RetainWriteScopes means a non-executing lane still owns write exclusion
// until its artifact/release gate is terminal. It is orthogonal to
// NonPreemptible: release ownership blocks overlapping work but must not
// become a preemption victim.
RetainWriteScopes bool
}
RunningLane is one already-running unit of work as the scheduler sees it for cross-tick accounting and preemption: the Ticket (for priority), the Process (liveness/identity), the ResourceGrants it holds (the reservation it charges against the ledger), and its MEASURED per-currency usage. The kernel's IndexedRunningSource builds these from the durable store.
type RunningSource ¶
type RunningSource interface {
RunningLanes() ([]RunningLane, error)
}
RunningSource yields the already-running lanes the scheduler must account for this tick — the cross-tick reservation accounting and preemption input. Like RunnableSource it must be an indexed read. A scheduler wired without a RunningSource does no cross-tick accounting (Running is empty) — correct only for a test or a cold start with nothing running.
func NewIndexedRunningSource ¶
func NewIndexedRunningSource(store Store) RunningSource
NewIndexedRunningSource returns the kernel's standard RunningSource over a Store, scoped to the DEFAULT cell.
func NewIndexedRunningSourceForCell ¶
func NewIndexedRunningSourceForCell(store Store, cell string) RunningSource
NewIndexedRunningSourceForCell returns a RunningSource scoped to a named cell.
func NewIndexedRunningSourceForCellWithQuarantineHook ¶
func NewIndexedRunningSourceForCellWithQuarantineHook( store Store, cell string, onQuarantine func(laneName string, reason PoisonLaneReason, detail string), ) RunningSource
NewIndexedRunningSourceForCellWithQuarantineHook is the production constructor that also wires a quarantine observability sink. The sink is called for every poison lane or orphan process the rebuild quarantines/drops, so the daemon can surface "the control plane survived a poison lane" as measured truth. The hook is advisory only — quarantine itself is a best-effort store write the running source performs regardless.
type Scheduler ¶
type Scheduler struct {
Refill RefillSource
// contains filtered or unexported fields
}
Scheduler is the thin loop wrapper around the colony decision core (§7.2, §15). It is the single scheduler: leader-elected via a cell-scoped ServiceLease (`scheduler/<cell>`), it runs ONE reconcile loop. A tick with no held lease is skipped (fail closed) — the structural cure for the v1 three-daemon split.
func NewScheduler ¶
func NewScheduler(cfg SchedulerConfig) *Scheduler
NewScheduler builds a Scheduler from its config.
func (*Scheduler) Holder ¶
Holder returns this scheduler instance's unique leader-election identity (the configured holder plus its per-instance suffix). It exists so a test or an operator can observe which concrete instance won leadership.
func (*Scheduler) PhaseStats ¶
func (s *Scheduler) PhaseStats() TickPhaseStats
PhaseStats returns the scheduler's per-phase tick-timer snapshot — the §11.2 question-1 instrumentation that splits a reconcile tick into ledger rebuild, ReconcileColony, write-scope acquire, and spawn so a load harness can name which sub-phase scales super-linearly.
func (*Scheduler) ReservedSlots ¶
ReservedSlots returns the total amount of a currency currently reserved by LIVE running lanes — Σ over running lanes of their grants for c. It is a read-only query (it does NOT schedule, admit, or mutate) the warm-pool re-tune uses to floor its target at live demand: the windowed admission rate forgets long-running lanes once their admission ages out of the window, so a re-tune keyed on the rate alone can size the warm target BELOW the slots those lanes still hold reserved — leaving Reservable (= warm Ready) below Reserved, which blocks all further admission and prevents any new admission from ever raising the rate again (the §7.5 re-tune deadlock). Flooring the target at this live reservation count keeps Reservable ≥ Reserved so admission can continue.
The fractional sum is rounded UP (math.Ceil), not truncated (review finding MEDIUM-11): a reservation of 16.5 truncated to 16 would let TuneWarmPool target only 17 (16 + its 1-slot buffer), leaving 17 − 16.5 = 0.5 of a slot free — too little to admit a new workspace_slot=1 request, starving the pool. Ceiling to 17 makes the integer floor cover the fractional remainder so a whole slot of headroom survives.
A nil RunningSource (cold start / test) yields 0, nil. A running-source read error is surfaced so the caller can keep the previous target rather than shrink against an unknown reservation count.
func (*Scheduler) Run ¶
Run drives the reconcile loop until stop is closed, ticking every interval. It is the §7.2 `for leader := range hold(ServiceLease{...})` shape: one loop, leader-elected, fail-closed on a missing lease.
A tick that fails for a real reason (store outage, runnable-read failure, handoff-barrier block, spawn failure) does NOT vanish: Run emits an EventSchedulerError to the journal so the supervision tree sees the outage. A clean non-leader skip (HeldLease false, Err nil) is normal and not reported.
func (*Scheduler) Tick ¶
func (s *Scheduler) Tick() TickResult
Tick runs exactly one reconcile pass (§7.2, §15). It:
- acquires (first tick) then INSTANCE-FENCED renews the cell-scoped scheduler ServiceLease and skips the tick if it cannot — fail closed. Only ErrLeaseHeld is a clean non-leader skip; any other error is a real outage and is surfaced. A renew that finds the lease lost (ErrNotHolder, or ErrNotFound when the native-TTL lease key has auto-deleted) re-elects rather than wedging — see electOrRenew.
- runs the §15.1 LEADERSHIP HANDOFF BARRIER: it reads the live offers and the durable running-lane state (grants + processes) and rebuilds the resource ledger from them BEFORE admitting. If that durable read fails, admission fails closed for this tick — the kernel never admits against an unknown ledger during a leadership-uncertainty window.
- reads runnable tickets via an indexed read (dependency-resolution fails closed on a lookup error).
- calls the pure ReconcileColony — cross-tick accounting + preemption.
- commits each bind as ONE atomic Store.Txn: ticket CAS, grants, scope lease, and lane. Process spawn is the only post-commit execution step.
- appends an Admission event to the journal.
type SchedulerConfig ¶
type SchedulerConfig struct {
Holder string
Leases *LeaseManager
Offers OfferSource
Runnable RunnableSource
// Running is the RunningSource for cross-tick reservation accounting and
// preemption. Optional — a nil Running means no running lanes are folded in
// (a cold start or a test). It MUST be cell-scoped to this scheduler's Cell
// (finding #7): production wires NewIndexedRunningSourceForCell(store, Cell)
// so a queen of cell B never folds cell A's running lanes into its ledger.
// NewIndexedRunningSource is the DefaultCell shorthand.
Running RunningSource
Spawner Spawner
// Preemptor enacts preemption decisions. Admissions that require preemption
// fail closed unless this is a concrete ReversiblePreemptor; a nil or
// non-reversible preemptor leaves the beneficiary visibly waiting.
Preemptor Preemptor
Journal Journal
// ExternalResourceRelease is an optional embedding-owned cleanup hook for
// non-kernel resources tied to a lane. The scheduler invokes it before
// finalizing preempted/interrupted lane-owned resources; a blocked result
// leaves the lane envelope intact for the owning controller to retry. Nil
// preserves the pure-kernel path.
ExternalResourceRelease ExternalResourceReleaseFunc
// Store is the object store the atomic admission transaction durably
// creates each admitted ticket's ResourceGrants into (finding #1: an
// admitted grant must SURVIVE the tick so the next ledger rebuild folds its
// reservation back in). It MUST be the same store the Runnable/Running
// sources read — a grant persisted to a different store is invisible to
// next-tick accounting. Optional: when nil the scheduler falls back to the
// LeaseManager's store, correct for the single-store kernel-MVP where the
// lease store IS the object store.
Store Store
// Cell is the §15.4 cell this scheduler is the queen of. The scheduler
// ServiceLease is `scheduler/<Cell>`. Empty defaults to DefaultCell.
Cell string
// GlobalStore is the bootstrap/global ticket-queue store used by named-cell
// schedulers to verify a migrated cell-local copy is still designated to
// this cell before admitting it. Nil preserves the single-store/default path.
GlobalStore Store
// LaneTargetPolicyName optionally names the LaneTargetPolicy object the
// scheduler's pre-admission budget gate reads. Empty means the documented
// unbounded budget fallback; no singleton discovery runs under the scheduler
// lease.
LaneTargetPolicyName string
// LeaseTTL bounds how long a crashed leader blocks a takeover. Defaults
// to 15s if zero.
LeaseTTL time.Duration
// ScopeLeaseTTL bounds how long a running lane's write-scope exclusion
// survives without scheduler renewal. It is deliberately separate from
// LeaseTTL: leadership should fail over quickly, while write ownership must
// tolerate scheduler jitter and distributed-store latency without making live
// lanes flicker out of /proc useful_lanes. Defaults to
// DefaultSchedulerScopeLeaseTTL(LeaseTTL) when zero.
ScopeLeaseTTL time.Duration
// AllowUnfencedSplitStore opts INTO running admission UNFENCED when the
// scheduler ServiceLease store (Leases) is a different backend from the
// object Store. A §15.1 fenced write is atomic with its fence only when the
// fence object and the write live in ONE store; with a split store the
// admission fence is dropped, so a stale queen that lost leadership mid-tick
// could still commit (Round-9 admission finding #2). NewScheduler therefore
// REJECTS a split store unless this is set. Production runs one store and
// never sets it; only a single-queen test that deliberately isolates the
// lease store (no competing leader, so the fence is moot) opts in.
AllowUnfencedSplitStore bool
// AdmissionGate is the OPTIONAL pre-admission hook (scheduler_admission_gate.go).
// Nil is fail-open — the scheduler behaves identically to today. The intent/
// acceptance gate (pkg/intent) is wired here to hold work that is incoherent
// with the principal's intent or needs a human's sign-off.
AdmissionGate AdmissionGate
// ExclusiveContentionTTL bounds how long fresh backpressure-exit evidence
// latches admission of an exclusive currency before the kernel spends ONE
// probe attempt to re-measure (exclusive_contention.go). Zero means
// DefaultExclusiveContentionTTL. It is also the worst-case admission delay
// after an out-of-band holder releases.
ExclusiveContentionTTL time.Duration
// ExclusiveIdleHoleThreshold is the inverse-watchdog window
// (exclusive_idle_watchdog.go): an exclusive currency continuously FREE
// while declared demand waits longer than this emits a typed severity-high
// journal event — the 91-minute-admission-hole class made observable. Zero
// means DefaultExclusiveIdleHoleThreshold.
ExclusiveIdleHoleThreshold time.Duration
// Clock is the injectable time source for the contention latch and the idle
// watchdog (tests compress windows without sleeping). Nil means the system
// clock. It does NOT replace lease/journal clocks — only the exclusive-
// currency invariants read it.
Clock func() time.Time
}
SchedulerConfig wires a Scheduler. holder is this kernel instance's identity for leader election.
type ScopeConflictGuard ¶
type ScopeConflictGuard interface {
AcquireScopeLease(lease *WriteScopeLease, cell string, fence *FenceRef) (Object, error)
ReleaseScopeLease(leaseName string, scope WriteScope, holder, cell, instanceID string) error
}
ScopeConflictGuard is the backend-owned WriteScopeLease primitive. It makes overlap detection and the lease write one atomic operation for stores that can guard more than one object at a time.
type ScopeKind ¶
type ScopeKind string
ScopeKind is the type of resource a WriteScopeLease locks. Only these four concrete kinds are lockable; a prose description is never lockable.
func (ScopeKind) IsLockable ¶
IsLockable reports whether a ScopeKind names a real lockable resource.
type Selector ¶
Selector filters a List. An empty Selector matches every object of the Kind. MatchLabels requires an exact match on each given label; the store evaluates it against the indexed ObjectMeta.Labels map, not a full deserialise scan.
type ServiceLease ¶
type ServiceLease struct {
ObjectMeta `json:",inline"`
Spec ServiceLeaseSpec `json:"spec"`
Status Status `json:"status"`
}
ServiceLease is a single-activation lease for a controller (leader election). Exactly one holder per (service, cell); see lease.go.
func (*ServiceLease) DeepCopyObject ¶
func (s *ServiceLease) DeepCopyObject() Object
DeepCopyObject returns an independent copy of the ServiceLease.
func (*ServiceLease) GetMeta ¶
func (s *ServiceLease) GetMeta() *ObjectMeta
GetMeta makes ServiceLease an Object.
type ServiceLeaseSpec ¶
type ServiceLeaseSpec struct {
Service string `json:"service"`
Holder string `json:"holder,omitempty"`
TTL time.Duration `json:"ttl"`
AcquiredAt time.Time `json:"acquiredAt,omitempty"`
RenewedAt time.Time `json:"renewedAt,omitempty"`
// InstanceID is the immutable per-acquisition identity token minted on a
// SUCCESSFUL acquire (a fresh take-over, not an idempotent renew). Renew
// and Release authenticate against this token, not the Holder string: a
// stale process reusing the same Holder string after its lease expired and
// a successor reacquired carries the OLD InstanceID and is rejected.
InstanceID string `json:"instanceID,omitempty"`
// Epoch is the §15.1/§15.5 LeadershipEpoch this lease grant carries. It
// increases monotonically across every fresh take-over of this service
// (carried forward unchanged on an idempotent renew). The queen stamps this
// epoch onto every correctness-bearing object it authors; a stale former
// leader holds an old epoch and is fenced (FenceEpoch).
Epoch int64 `json:"epoch,omitempty"`
}
ServiceLeaseSpec is the desired-state of a ServiceLease.
type SnapshotWatcher ¶
type SnapshotWatcher interface {
WatchSnapshot(kind Kind, sel Selector, limit int, stop <-chan struct{}) (ListPage, <-chan WatchEvent, error)
}
SnapshotWatcher is the additive watch extension for stores that can capture a bounded selector snapshot and register the live watcher atomically. The returned page is the initial state at ResourceVersion; the returned channel streams later changes and does not replay the page. If the matching set exceeds limit, NextContinue is non-empty and the live channel is nil.
type Spawner ¶
Spawner is the kernel side-effect the scheduler loop invokes for each admission: it must write a Process row (journaled) BEFORE any external work starts, so 100% of lanes have a Process row — the spawn-visibility SLO (§11).
type SpawnerStore ¶
type SpawnerStore interface {
SchedulerStore() Store
}
SpawnerStore is an OPTIONAL seam a Spawner may implement to let NewScheduler verify it is backed by the SAME object store the bind transaction commits the Lane/grants into. A Spawner whose store differs would never find the bound Lane on the post-commit SpawnLane (Round-10 admission #3). A Spawner that does not implement this is unchecked (a test stub with no store of its own).
type Status ¶
type Status struct {
Conditions []Condition `json:"conditions,omitempty"`
ObservedGeneration int64 `json:"observedGeneration"`
LastTransitionAt time.Time `json:"lastTransitionAt"`
// WaitingOn names the resource currencies (or dependency tickets) that
// currently block this object. Typed and visible in /proc; the scheduler
// sets it on a Ticket that admission rejected.
WaitingOn []string `json:"waitingOn,omitempty"`
// Phase is the kind-specific lifecycle state (e.g. lane state machine).
Phase string `json:"phase,omitempty"`
// TerminalEvidence is the bounded record of WHY a worker exited non-zero,
// captured by the runtime on a lane's terminal failure. It is nil for every
// non-failed object and for non-lane kinds — a Lane is the only kind whose
// worker can exit — so the `omitempty` pointer adds zero JSON/ABI surface
// when absent and round-trips schemalessly through the store's JSON codec.
// It surfaces in /proc and feeds the consecutive-failure circuit-breaker.
TerminalEvidence *TerminalEvidence `json:"terminalEvidence,omitempty"`
// ActivitySummary is the ONE durable per-attempt aggregate of a lane
// worker's observed activity (tokens, tool calls, turns, rate-limit
// standing — agentos.agent_activity.v1), CAS-written at attempt stop. Nil
// for every non-lane kind and for attempts that produced no observed
// activity data; like TerminalEvidence the `omitempty` pointer adds zero
// ABI surface when absent. See object_activity_summary.go.
ActivitySummary *ActivitySummary `json:"activitySummary,omitempty"`
}
Status is the kernel-observed state of an object. ObservedGeneration records the Generation a controller last reconciled, so a stale controller is detectable (ObservedGeneration < Generation).
func CloneStatus ¶
CloneStatus returns an independent deep copy of a Status — the Conditions and WaitingOn slices are freshly allocated. It is the exported counterpart to CloneMeta: a kind embedding kernel.Status (directly or via a kind-specific status struct) deep-copies it with one kernel-owned helper. The kernel's own kinds route their unexported clone() through it.
type Store ¶
type Store interface {
// Get returns an independent copy of the object, or ErrNotFound.
Get(kind Kind, name string) (Object, error)
// Create stores a new object, stamping UID, ResourceVersion, and
// Generation. It returns ErrAlreadyExists for a duplicate (Kind, Name).
Create(obj Object) (Object, error)
// Update compare-and-swaps on ResourceVersion. On success it bumps the
// version (and Generation if the spec changed conceptually — callers
// bump Generation themselves before calling). On a stale version it
// returns a *ConflictError.
Update(obj Object) (Object, error)
// Delete removes the object, or returns ErrNotFound. Delete is a BLIND
// delete — it does NOT compare ResourceVersion, so a stale actor can erase
// an object a concurrent writer just updated. Use it ONLY for
// non-coordination cleanup (GC of an object no other actor races on). For
// a delete that must lose cleanly against a concurrent update, use
// DeleteCAS (a version-guarded delete) — CAS is the kernel's concurrency
// primitive and a versioned object should not be blindly deleted.
Delete(kind Kind, name string) error
// List returns copies of every object of the Kind matching the Selector.
// It is an indexed read, not a full deserialise scan.
List(kind Kind, sel Selector) ([]Object, error)
// ListPage returns a bounded page for the same selector. Continue is the
// opaque token returned by the prior call for this kind and selector.
ListPage(kind Kind, sel Selector, opts ListOptions) (ListPage, error)
// Watch returns a buffered channel of changes for the Kind. The channel
// first replays the current state as WatchAdded events, then streams
// live changes. Cancelling stop closes the channel and frees resources.
Watch(kind Kind, stop <-chan struct{}) (<-chan WatchEvent, error)
// Txn atomically commits all ops iff every condition holds. If any
// condition or op precondition fails, no object is mutated.
Txn(conds []Cond, ops []Op) error
}
Store is the object-store ABI. The kernel works exclusively against this interface; the etcd backend (a later phase) and MemStore both implement it.
Update is a compare-and-swap on ResourceVersion: the supplied object's ResourceVersion must equal the stored version or Update returns a *ConflictError. Txn is the multi-object extension of that same rule: no object is silently overwritten, and related writes can share one atomic condition boundary.
type StoreLabelNormalizer ¶
type StoreLabelNormalizer interface {
NormalizeStoreLabels()
}
StoreLabelNormalizer lets extension objects mirror typed spec/status fields onto indexed labels without store backends importing extension packages.
type StoreStats ¶
type StoreStats struct {
// CASConflicts is the number of Update / DeleteVersioned calls that lost
// the compare-and-swap on ResourceVersion — a real measure of write
// contention (the §7.7 / §11.2 store CAS-retry signal).
CASConflicts int64 `json:"casConflicts"`
}
StoreStats is a snapshot of a MemStore's internal instrumentation counters.
type TerminalEvidence ¶
type TerminalEvidence struct {
// ExitCode is the worker process's exit status (non-zero on failure).
ExitCode int `json:"exitCode"`
// Attempt is the Lane.Spec.Attempt the evidence was captured for. Evidence
// is written per ATTEMPT onto the shared lane object and a clean later
// attempt writes nothing, so without this stamp a stale attempt-1 record
// (e.g. one rc=87 backpressure exit) would classify the LANE's terminal
// outcome forever — an uncapped requeue loop. Classifiers must honour the
// evidence only when it describes the lane's final attempt; 0 means the
// record predates the stamp (legacy) and is trusted as before.
Attempt int `json:"attempt,omitempty"`
// Reason is a short, generic explanation. It is the worker-provided
// structured reason when present (the optional terminal_status.json), else a
// synthesised "exit_code=<n>". It never contains product-specific schema.
Reason string `json:"reason,omitempty"`
// StderrTail is the bounded tail (<=~2 KB / <=40 lines) of the worker's
// stderr — the operator's first diagnostic. Bounded by construction so the
// object can never store an unbounded log.
StderrTail string `json:"stderrTail,omitempty"`
// Signature is a short stable hash of (ExitCode + a NORMALISED Reason): the
// reason with digits, paths, and timestamps stripped, so "iter 112 OOM" and
// "iter 208 OOM" hash the SAME. Two genuinely-identical failures share a
// signature (the circuit-breaker increments toward its cap); a varied failure
// gets a new signature (the cap resets and tolerates it). Empty only on the
// zero value.
Signature string `json:"signature,omitempty"`
// OutcomeClass is the TYPED terminal outcome (see ClassifyOutcome). It is the
// load-bearing distinction that makes recovery class-appropriate instead of
// "any non-zero exit is a failure": BACKPRESSURE requeues without counting
// toward the respawn cap, FAILED hits the bounded respawn machine, KILLED is a
// signal death. Derived once, in the single constructor, so the producer and
// the redrive cannot disagree.
OutcomeClass string `json:"outcomeClass,omitempty"`
// ObservedAt is when the runtime captured the evidence (lane terminal time).
ObservedAt time.Time `json:"observedAt"`
}
TerminalEvidence is the bounded, domain-agnostic record of a worker's non-zero exit, captured by the runtime and surfaced on the lane. It is the kernel's only knowledge of a terminal failure's cause: an exit code, a generic reason, a bounded stderr tail, and a stable signature the circuit-breaker keys on. It is never populated for a clean exit.
func NewTerminalEvidence ¶
func NewTerminalEvidence(exitCode int, reason, rawStderr string, observedAt time.Time) *TerminalEvidence
NewTerminalEvidence builds a TerminalEvidence from an exit code, a generic reason, and a raw stderr blob, computing the bounded tail and the stable signature. reason "" is replaced by the synthesised "exit_code=<n>" so the record always carries a non-empty cause. observedAt is the capture time. It is the single constructor the runtime uses, so the signature derivation is defined in exactly one place (producer and circuit-breaker cannot disagree).
func NewTerminatedEvidence ¶
func NewTerminatedEvidence(reason string, observedAt time.Time) *TerminalEvidence
NewTerminatedEvidence builds the TerminalEvidence for a lane an OPERATOR terminate signal stopped. OutcomeClass is OutcomeTerminated by construction — derived here, in the single constructor, so the lane-execution producer and the recovery consumers cannot disagree about what an operator stop looks like. The exit code is the -1 "did not run to completion" sentinel: the process was stopped on purpose, so whatever signal-death code it happened to die with is not the cause and must not feed the failure circuit-breaker.
type TerminalLaneOwnedResourceTxnPlanOption ¶
type TerminalLaneOwnedResourceTxnPlanOption func(*terminalLaneOwnedResourceTxnPlanConfig)
TerminalLaneOwnedResourceTxnPlanOption tunes terminal lane resource planning.
func WithTerminalLaneOwnedResourceExternalProof ¶
func WithTerminalLaneOwnedResourceExternalProof() TerminalLaneOwnedResourceTxnPlanOption
WithTerminalLaneOwnedResourceExternalProof allows externally-owned currencies in a terminal lane grant to be deleted in the caller's transaction. The caller is responsible for pairing the transaction with the relevant application/provider absence or release proof.
type TickLaneSnapshot ¶
type TickLaneSnapshot struct {
// contains filtered or unexported fields
}
TickLaneSnapshot is the tick-local view of the cell's non-terminal lanes plus the merge-ready retained-scope lanes. It is built once at tick start and MUTATED by the recovery step: a consumer that terminalizes a lane prunes it, so later consumers in the same tick never act on a lane the tick already closed (their CAS writes would lose anyway; pruning removes the wasted work). Not safe for concurrent use — the tick is single-threaded.
func (*TickLaneSnapshot) RunningPreloadLanes ¶
func (t *TickLaneSnapshot) RunningPreloadLanes() []*Lane
RunningPreloadLanes exposes the snapshot view the running source's preload needs: every active-phase lane plus the merge-ready retained set.
type TickLaneSnapshotConsumer ¶
type TickLaneSnapshotConsumer interface {
UseTickLaneSnapshot(lanes []*Lane)
}
TickLaneSnapshotConsumer is the OPTIONAL seam a RunningSource implements to reuse the tick's shared snapshot instead of issuing its own preload lists. The scheduler offers the snapshot before each collectRunning; a source that does not implement it keeps its own reads (compatibility: tests, embedders). A wrapper source (the daemon's preemption guard) forwards to its base.
type TickPhase ¶
type TickPhase string
TickPhase names one timed sub-phase of a scheduler reconcile tick. The §11.2 question-1 instrumentation breaks a tick into these four so a super-linear metric is attributable to a stage, not just "the scheduler".
const ( // PhaseLedgerRebuild is the §15.1 handoff-barrier ledger rebuild — reading // measured offers and folding durable running-lane state into the ledger. PhaseLedgerRebuild TickPhase = "ledger_rebuild" // PhaseReconcile is the pure ReconcileColony decision core — priority order, // admission, preemption. PhaseReconcile TickPhase = "reconcile_colony" // PhaseScopeAcquire is write-scope-lease acquisition for the admitted set. PhaseScopeAcquire TickPhase = "write_scope_acquire" // PhaseSpawn is enacting the decision — spawning admitted lanes (a Process // row first) and preempting victims. PhaseSpawn TickPhase = "spawn" )
type TickPhaseStats ¶
type TickPhaseStats struct {
Phases []PhaseStat `json:"phases"`
}
TickPhaseStats is the §11.2 per-phase scheduler-tick instrumentation: one PhaseStat per timed sub-phase, in canonical order.
func (TickPhaseStats) Phase ¶
func (t TickPhaseStats) Phase(p TickPhase) PhaseStat
Phase returns the PhaseStat for a named sub-phase, or a zero PhaseStat.
type TickResult ¶
type TickResult struct {
// HeldLease is false when the tick was skipped because this instance is
// not the elected leader (fail closed).
HeldLease bool
Admitted []Admission
Waiting []Ticket
Preempted []Preemption
EffectiveTarget int
// Epoch is the LeadershipEpoch this tick ran under (0 when no lease held).
Epoch int64
Err error
}
TickResult is what one reconcile tick produced — returned so the loop owner (and tests) can observe the decision without reaching into the kernel.
type Ticket ¶
type Ticket struct {
ObjectMeta `json:",inline"`
Spec TicketSpec `json:"spec"`
Status Status `json:"status"`
}
Ticket is a unit of desired work — the canonical org queue entry. It carries its Tier, Scope, a typed WriteScope (never a prose string), a ResourceRequest the scheduler admits against, and dependency edges.
func (*Ticket) DeepCopyObject ¶
DeepCopyObject returns an independent copy of the Ticket.
func (*Ticket) GetMeta ¶
func (t *Ticket) GetMeta() *ObjectMeta
GetMeta and DeepCopyObject make Ticket an Object.
type TicketSpec ¶
type TicketSpec struct {
Tier Tier `json:"tier"`
Scope string `json:"scope"`
WriteScope []WriteScope `json:"writeScope,omitempty"`
ResourceRequest ResourceRequest `json:"resourceRequest"`
DependsOn []string `json:"dependsOn,omitempty"`
Unblocks []string `json:"unblocks,omitempty"`
// CreatedSeq is a monotonic submission counter used as the fair-share
// tie-break within a tier (older ticket admits first).
CreatedSeq int64 `json:"createdSeq,omitempty"`
// Command is the launchable argv (program first) for DIRECT-DISPATCH. It is
// optional: absent means the ticket declares nothing to launch (admit-as-decided),
// byte-identical to a pre-direct-dispatch ticket. When the kernelv2 executor owns
// the work, this becomes the WorkSpec.Command the Launcher spawns.
Command []string `json:"command,omitempty"`
// Env is the additional process environment (KEY=VALUE) applied on top of the
// daemon's inherited env when the ticket carries a Command. Optional.
Env []string `json:"env,omitempty"`
// Workspace is an explicit working directory the Launcher spawns the process
// into. Empty means the executor allocates its own per-work ephemeral workdir.
Workspace string `json:"workspace,omitempty"`
}
TicketSpec is the desired-state of a Ticket. DependsOn/Unblocks are dependency edges (other Ticket Names); a Ticket with an unresolved DependsOn is not runnable.
type Tier ¶
type Tier string
Tier selects both the cost tier and the engineer seniority of a Ticket; routing, retries, and resource gating all key off it. t4 is the highest priority (principal / founder sign-off), t0 the lowest (junior).
type ValidatableObject ¶
type ValidatableObject interface {
// Validate reports a content-correctness error that must reject the object
// at the store write path. A nil return permits the write. It is called on
// every Create and Update, so it must be cheap and side-effect-free.
Validate() error
}
ValidatableObject is the OPTIONAL self-validation seam of the store write path. A durable kind — a kernel-owned kind or an extension kind declared outside pkg/kernel — that needs a content-correctness rule the generic envelope checks cannot express implements it, and validateForWrite calls Validate() on every Create/Update before the object reaches the store.
This is how an EXTENSION kind enforces a write-path invariant the kernel itself does not know about (the §16.3 policy kinds carry Go-int fields whose proto wire form is int32 — a value outside int32 range must be rejected at the write path, never silently truncated on the wire). pkg/kernel cannot import pkg/policy to validate a policy kind directly, so the policy kind carries the rule itself and the generic gate invokes it through this interface — both the gRPC PutLaneTargetPolicy path and the file boot loader go through store.Create/Update, so one Validate() implementation gates both.
It is purely additive to the frozen ABI: it adds a new interface and never changes an existing method. A kind that does NOT implement it is unaffected — validateForWrite simply skips the Validate() step for it.
type Verdict ¶
type Verdict struct {
OK bool
Grants []ResourceGrant
BlockedOn []string
}
Verdict is the result of an admission decision. OK is true only if every currency in the request fits the ledger's remaining headroom. On rejection, BlockedOn names the failing currencies (typed, surfaced in /proc); Grants is nil. On success, Grants holds one ResourceGrant per non-empty request — the reservation the kernel issues, owned later by the spawned Process.
func AdmitAndReserve ¶
func AdmitAndReserve(req ResourceRequest, ledger *ResourceLedger) (Verdict, error)
AdmitAndReserve is the convenience path for a caller that wants the decide-and-charge done together (e.g. a controller admitting a single out-of-band request). On an OK verdict it reserves against the ledger and returns the verdict; on rejection it returns the verdict unchanged and the ledger is untouched. The scheduler does NOT use this — it keeps the two steps apart so Reconcile is pure — but it makes the common single-shot case safe.
func TryAdmit ¶
func TryAdmit(req ResourceRequest, ledger *ResourceLedger) Verdict
TryAdmit is the admission-control gate (§7.3). It admits a request ONLY if, for EVERY currency c, req[c] ≤ Reservable[c] − Reserved[c] — all currencies simultaneously, the single gate v1 never had.
TryAdmit does NOT mutate the ledger. The scheduler, on an OK verdict, calls ledger.Reserve(req) itself before spawning — keeping "decide" and "charge" separate so the decision logic stays a pure function (see scheduler.go's Reconcile). On rejection the verdict's BlockedOn names exactly the short currencies, in canonical order, so /proc shows why a ticket is waiting.
The grant in an OK verdict carries a placeholder Name; Reconcile re-stamps it with a durable-stable name via stampGrants (see grantName). TryAdmit cannot name the grant durably itself — it does not know the ticket — so naming is the caller's job and TryAdmit stays a pure currency check.
type WaitCondition ¶
type WaitCondition struct {
// Kind classifies the awaited terminal state.
Kind WaitConditionKind
// Subject is the object Name the condition concerns — the lane name, the
// process name, the ticket name, or a batch id. The satisfying journal
// event carries the same Subject.
Subject string
}
WaitCondition is the §15.6 condition a waiter registers on. It is satisfied by a durable journal event whose Type maps to Kind and whose Subject equals Subject — so the condition's truth is the durable journal, never a transient signal.
type WaitConditionKind ¶
type WaitConditionKind string
WaitConditionKind classifies what a §15.6 Wait is waiting on. The kind plus a Subject form the condition; a journal event of the matching Type+Subject satisfies it.
const ( // WaitLaneComplete — a durable Lane reached a terminal state (DONE, // ESCALATED). Satisfied by an EventLaneProgress(DONE) / EventEscalation. WaitLaneComplete WaitConditionKind = "LaneComplete" // WaitProcessExit — a Process terminated. Satisfied by an EventProcessStatus // carrying a terminal phase. This is the §15.6 point-4 supervision case // (Akka DeathWatch / Erlang monitor). WaitProcessExit WaitConditionKind = "ProcessExit" // WaitVerifierVerdict — a verifier wrote a terminal pass/fail verdict. WaitVerifierVerdict WaitConditionKind = "VerifierVerdict" // WaitDependencyCleared — a ticket's blocking dependency cleared. WaitDependencyCleared WaitConditionKind = "DependencyCleared" // WaitBatchConverged — a fan-in completion: a set of children all reached a // terminal state (the §15.6 point-5 fan-in Wait). WaitBatchConverged WaitConditionKind = "BatchConverged" )
type WaitHandle ¶
type WaitHandle struct {
// contains filtered or unexported fields
}
WaitHandle is a registered §15.6 waiter. The caller blocks on it with Wait, which resolves from the edge-trigger notification OR the journal-reconcile backstop, then releases the registration.
func (*WaitHandle) Cancel ¶
func (h *WaitHandle) Cancel()
Cancel retires a waiter that no longer needs its result (the caller gave up, or resolved another way). It is idempotent.
func (*WaitHandle) FireCount ¶
func (h *WaitHandle) FireCount() int64
FireCount returns how many times this waiter's resolution was attempted — the §15.6 exactly-once assertion surface. It MUST be exactly 1 for a resolved waiter: a notify racing the reconcile backstop bumps it past 1, but resolve's CAS still delivers only once. A test asserts on it.
func (*WaitHandle) Wait ¶
func (h *WaitHandle) Wait(timeout time.Duration) (WaitResult, bool)
Wait blocks until the §15.6 condition is satisfied OR the timeout elapses. It resolves in priority order: the edge-trigger notification if it arrived, else the journal-reconcile backstop (a scan of the durable journal for the satisfying event). ok is false only if neither the notification nor the journal shows the condition satisfied inside the timeout.
This is the §15.6 informer rule made into one call: notification optimises latency, the journal guarantees correctness, reconcile is the backstop.
Wait RELEASES the registration before it returns — on the notification path, the timeout path, and the reconcile path alike. SatisfyCondition removes a waiter from registry.waiters when it fires the edge-trigger, but the timeout and the reconcile-resolution paths do not; without this release a timed-out or reconcile-resolved waiter would linger in registry.waiters forever — it would still count in Stats().Pending and could be resolved AGAIN later as a duplicate (§15.6 Finding 5). The delete-by-id is idempotent, so removing an already-removed waiter is harmless.
func (*WaitHandle) Waiter ¶
func (h *WaitHandle) Waiter() string
Waiter returns the waiter id this handle registered.
type WaitNotifier ¶
type WaitNotifier interface {
// NotifyWaiter delivers a completion notification for a satisfied condition
// to the named waiter. It is point-to-point and correctness-bearing in
// transport, but a delivery failure is non-fatal: the journal/reconcile
// backstop still resolves the Wait.
NotifyWaiter(waiter string, result WaitResult) error
}
WaitNotifier is the fast point-to-point notification path of §15.6 point 2. It is satisfied by the §15.3 hive command/reply plane; the kernel keeps it an interface so pkg/kernel never imports pkg/hive. Notify is best-effort — the journal is the durable backstop, so a notifier error never fails SatisfyCondition.
type WaitRegistry ¶
type WaitRegistry struct {
// contains filtered or unexported fields
}
WaitRegistry is the kernel-side §15.6 ABI: it registers waiters on conditions, satisfies a condition by writing a durable journal event FIRST and then firing the registered waiters, and exposes the journal-reconcile backstop for a waiter that missed the edge-trigger.
It is the durable-correctness owner. The journal is the truth (the satisfying event is written before any notification); the WaitNotifier is the latency optimisation; reconcile is the backstop. Exactly-once from the waiter's perspective is enforced by waiterEntry.resolve.
func NewWaitRegistry ¶
func NewWaitRegistry(journal Journal, notifier WaitNotifier, clock Clock) *WaitRegistry
NewWaitRegistry builds a §15.6 Wait registry over the kernel journal (the durable backstop) and an optional WaitNotifier (the §15.3 fast path). A nil notifier means notification-less operation — every Wait still resolves via the journal-reconcile backstop, which is the §15.6 correctness guarantee. clock may be nil for the system clock.
func (*WaitRegistry) ReconcileCondition ¶
func (r *WaitRegistry) ReconcileCondition(condition WaitCondition) (Event, bool)
ReconcileCondition is the §15.6 restart-replay backstop as a standalone ABI call: it scans the durable journal for the event satisfying a condition, independent of any registered waiter. A supervisor resuming after a crash uses it to learn which children already terminated (the §15.6 point-5 crash-safe fan-in). It returns the satisfying event and whether one was found.
func (*WaitRegistry) RegisterWaiter ¶
func (r *WaitRegistry) RegisterWaiter(waiter string, condition WaitCondition) *WaitHandle
RegisterWaiter registers a §15.6 waiter on a condition and returns a handle to block on. If the condition was ALREADY satisfied (its terminal journal event already landed) the handle resolves immediately from the recorded event — a waiter that registers late never misses a completion.
This is the kernel ABI surface §15.6 requires: "register a waiter on a condition; the condition is satisfied by a durable journal event".
func (*WaitRegistry) SatisfyCondition ¶
func (r *WaitRegistry) SatisfyCondition(condition WaitCondition, ev Event) int64
SatisfyCondition is the §15.6 kernel-side completion path. It performs the mandated ordering EXACTLY:
- Write the terminal state DURABLY FIRST — append the satisfying event to the journal. The journal is the truth and the backstop, so it must land before any notification.
- Record the condition as satisfied (so a waiter that registers later resolves immediately, not forever-blocked).
- ONLY THEN deliver the point-to-point completion notification to every registered waiter — the edge-trigger — and, for each, ask the optional WaitNotifier to push it over the §15.3 command/reply fast path.
The terminal event the caller passes is appended as-is (its Offset/Timestamp stamped by the journal). Returns the journal offset of the durable event.
func (*WaitRegistry) Stats ¶
func (r *WaitRegistry) Stats() WaitStats
Stats returns the §15.6 registry counters — observability + the exactly-once instrumentation surface.
type WaitResult ¶
type WaitResult struct {
// Condition is the condition that was satisfied.
Condition WaitCondition
// Event is the durable journal event that satisfied it — the authority.
Event Event
// ViaReconcile is true when the result came from the journal-reconcile
// backstop rather than the edge-trigger notification (the waiter missed the
// notification but the journal still resolved it — §15.6 point 3).
ViaReconcile bool
// ObservedAt is when the registry resolved the condition.
ObservedAt time.Time
}
WaitResult is what a satisfied §15.6 Wait delivers to its waiter.
type WaitStats ¶
type WaitStats struct {
// Registered is the total waiters ever registered.
Registered int64 `json:"registered"`
// NotifyFires is the resolutions delivered via the edge-trigger.
NotifyFires int64 `json:"notifyFires"`
// ReconcileFires is the resolutions delivered via the journal backstop.
ReconcileFires int64 `json:"reconcileFires"`
// Pending is the waiters still registered (unresolved right now).
Pending int `json:"pending"`
}
WaitStats is a snapshot of the §15.6 registry counters.
type WatchEvent ¶
type WatchEvent struct {
Type WatchEventType
Object Object
}
WatchEvent is one change delivered on a Watch channel. Object is an independent copy — the receiver may mutate it freely.
type WatchEventType ¶
type WatchEventType string
WatchEventType classifies a change emitted on a Watch channel.
const ( // WatchAdded is the initial state of an object plus every later Create. WatchAdded WatchEventType = "ADDED" // WatchModified is emitted on every Update. WatchModified WatchEventType = "MODIFIED" // WatchDeleted is emitted on every Delete; Object holds the last state. WatchDeleted WatchEventType = "DELETED" )
type WriteScope ¶
WriteScope is a typed exclusive-write claim. Kind selects the lock type and Value the concrete resource (a path prefix, a branch name, a program id, a shard id). A WriteScope whose Kind is not lockable is rejected by the lease manager — prose strings cannot be leased.
type WriteScopeLease ¶
type WriteScopeLease struct {
ObjectMeta `json:",inline"`
Spec WriteScopeLeaseSpec `json:"spec"`
Status Status `json:"status"`
}
WriteScopeLease is a typed exclusive lock over a path / branch / program / benchmark_shard. See lease.go for prefix-overlap collision semantics.
func (*WriteScopeLease) DeepCopyObject ¶
func (w *WriteScopeLease) DeepCopyObject() Object
DeepCopyObject returns an independent copy of the WriteScopeLease.
func (*WriteScopeLease) GetMeta ¶
func (w *WriteScopeLease) GetMeta() *ObjectMeta
GetMeta makes WriteScopeLease an Object.
type WriteScopeLeaseIdentity ¶
type WriteScopeLeaseIdentity struct {
Scope WriteScope `json:"scope"`
Holder string `json:"holder"`
Cell string `json:"cell,omitempty"`
InstanceID string `json:"instanceID"`
}
WriteScopeLeaseIdentity is the exact acquisition identity of one WriteScopeLease. It is copied onto Process.Spec so lane cleanup can release scopes even when the Ticket that originally declared WriteScope is gone.
type WriteScopeLeaseSpec ¶
type WriteScopeLeaseSpec struct {
Scope WriteScope `json:"scope"`
Holder string `json:"holder"`
TTL time.Duration `json:"ttl"`
AcquiredAt time.Time `json:"acquiredAt"`
// InstanceID is the immutable per-acquisition identity token, minted on a
// fresh acquire. Release authenticates against it, not the Holder string.
InstanceID string `json:"instanceID,omitempty"`
// Epoch is the LeadershipEpoch the issuing queen held. A scope lease issued
// by a stale queen is fenced.
Epoch int64 `json:"epoch,omitempty"`
}
WriteScopeLeaseSpec is the desired-state of a WriteScopeLease.
Source Files
¶
- admission.go
- admission_scope_budget.go
- admission_txn.go
- admission_txn_bind.go
- admission_txn_fence.go
- admission_txn_rollback.go
- cell_directory.go
- cell_placement.go
- cell_placement_copy.go
- cell_placement_cursor.go
- cell_placement_lease.go
- cell_placement_markers.go
- cell_placement_ownership.go
- cell_placement_status.go
- currency_registry.go
- exclusive_contention.go
- exclusive_idle_watchdog.go
- journal.go
- journal_cursor.go
- journal_interface.go
- journal_metrics.go
- journal_retention.go
- lane_execution_resource_release.go
- lane_published_artifact.go
- lane_resource_fence.go
- lane_resource_provider_token_release.go
- lane_resource_release.go
- lane_resource_release_atomic.go
- lane_resource_release_terminal.go
- lane_terminal.go
- lane_terminal_resource_txn_plan.go
- lane_terminate.go
- lease.go
- lease_instance.go
- lease_scope.go
- lease_scope_default.go
- lease_scope_identity.go
- object.go
- object_activity_summary.go
- object_apps.go
- object_leases.go
- object_resource.go
- object_runtime.go
- object_terminal_evidence.go
- object_tickets.go
- object_topology.go
- registry.go
- resource.go
- resource_integral.go
- retry_policy.go
- scheduler.go
- scheduler_admission_gate.go
- scheduler_agent_stop.go
- scheduler_budget.go
- scheduler_budget_heap.go
- scheduler_config.go
- scheduler_lease.go
- scheduler_lifecycle_ledger.go
- scheduler_loop.go
- scheduler_metrics.go
- scheduler_pause.go
- scheduler_pending_timeout.go
- scheduler_poison_lane.go
- scheduler_reconcile.go
- scheduler_reconcile_batched.go
- scheduler_recovery.go
- scheduler_refill.go
- scheduler_running_preload.go
- scheduler_running_source.go
- scheduler_scope_decision.go
- scheduler_scope_reconcile.go
- scheduler_struct.go
- scheduler_tick.go
- scheduler_tick_snapshot.go
- scheduler_waiting.go
- scope_guard.go
- scope_set.go
- selector.go
- store.go
- store_batched_get.go
- store_fence.go
- store_fence_mem.go
- store_index.go
- store_page.go
- store_scope_index.go
- store_txn.go
- store_validate.go
- store_watch_from.go
- store_write_ops.go
- wait.go
- write_scope_cover.go