watchdog2

README

Watchdog2

Scans the discovery cache and takes inventory of available services. This inventory is compared with the expected inventory and compared against two thresholds: yellow and red. Red and yellow correspond to different pagerduty API tokens.

Known issues:

• The watchdog pings pagerduty with updates to the ticket on every scan during an incident. Updates to degraded actions are not sent to pagerduty once the initial description has been sent. If a system becomes more degraded after the initial alert we will not become aware through slack. We could solve this by creating multiple incidents and tracking them separately. The service degradation from Yellow to Red does work properly -- the Yellow incident is closed and the Red is opened.

• Updates to degraded actions are compared with the last known good inventory. When the host is fixed a new service will come up but with a different ID. The watchdog will count the new service but still count the old service as missing. Minor detail that can be confusing to those who are looking at alerts.

• Many services are missing a version (as evidenced from plaintext examination of the discovery cache). Their versions are set to "~0" in the tool.

• The scamp library used by the watchdog does not have consistent support for processing data off io.Readers, most API calls expect filepaths and will open files. Those API calls are difficult to stub in tests and thus the test suit for watchdog2 is limited. The scamp library calls which take filepaths should be split in to fileopening and stream processing calls, then the watchdog test suit could be expanded.

Documentation

Index

• func ResolveEvent(pagerdutyKey, incidentId, reason string) (err error)
• func TriggerEvent(pdKey, description string, details interface{}) (key string, err error)
• type DegradedAction
• type DegradedService
• type ExpectedInventory
  • func LoadExpectedInventoryFromFile(path string) (ei ExpectedInventory, err error)
  • func NewExpectedInventory() (ei ExpectedInventory)
  • func (ei ExpectedInventory) GetSectorHealth(inv *Inventory) (secH SectorHealth)
  • func (ei ExpectedInventory) GetSystemHealth(inv *Inventory) (sh *SystemHealth)
• type ExpectedInventoryEntry
• type Incident
• type Inventory
  • func NewInventory(cachePath string) (i *Inventory, err error)
  • func (i *Inventory) Clone() (i2 *Inventory)
  • func (old *Inventory) Diff(inv *Inventory) (diff map[string]InventoryDiffEntry)
  • func (i *Inventory) Get(mangledName string) (val int, ok bool)
  • func (i *Inventory) GetList(mangledName string) (list []ServiceDesc, ok bool)
  • func (i *Inventory) Reload() (err error)
• type InventoryDiffEntry
• type InventoryType
• type PagerDutyEvent
• type PagerDutyEventContext
• type PagerdutyIncident
  • func NewPagerdutyIncident(pagerdutyKey string, missing SectorHealth) (yi *PagerdutyIncident)
  • func (pi *PagerdutyIncident) Description() (desc string, err error)
  • func (pi *PagerdutyIncident) Resolve() error
• type SectorDigestMap
• type SectorHealth
  • func NewSectorHealth() SectorHealth
  • func (sh SectorHealth) IsDegraded() bool
  • func (sh SectorHealth) Red() (red SectorHealth)
  • func (sh SectorHealth) ToSectorHealthByInventory() (shi SectorHealthByInventory)
  • func (sh SectorHealth) Yellow() (red SectorHealth)
• type SectorHealthByInventory
  • func NewSectorHealthByInventory() (shi SectorHealthByInventory)
  • func (shi SectorHealthByInventory) Add(sector, hostName, serviceName string, ...)
• type ServiceDesc
• type SlackIncident
  • func NewSlackIncident(expInv *ExpectedInventory, inv *Inventory) (yi *SlackIncident)
• type SystemHealth
  • func NewSystemHealth() (sh *SystemHealth)
  • func (sh *SystemHealth) IsDegraded() bool
  • func (sh *SystemHealth) MarkMissing(mangledName string)
  • func (sh *SystemHealth) MarkRed(mangledName string)
  • func (sh *SystemHealth) MarkYellow(mangledName string)

Functions

func ResolveEvent

func ResolveEvent(pagerdutyKey, incidentId, reason string) (err error)

func TriggerEvent

func TriggerEvent(pdKey, description string, details interface{}) (key string, err error)

Types

type DegradedAction

type DegradedAction struct {
	ActionName       string // class:method~version
	MinHealthyCount  int
	CurrentCount     int
	HealthLabel      string // red/yellow
	MissingInstances []DegradedService
}

type DegradedService

type DegradedService ServiceDesc

type ExpectedInventory

type ExpectedInventory map[string]ExpectedInventoryEntry

func LoadExpectedInventoryFromFile

func LoadExpectedInventoryFromFile(path string) (ei ExpectedInventory, err error)

func NewExpectedInventory

func NewExpectedInventory() (ei ExpectedInventory)

func (ExpectedInventory) GetSectorHealth

func (ei ExpectedInventory) GetSectorHealth(inv *Inventory) (secH SectorHealth)

func (ExpectedInventory) GetSystemHealth

func (ei ExpectedInventory) GetSystemHealth(inv *Inventory) (sh *SystemHealth)

type ExpectedInventoryEntry

type ExpectedInventoryEntry struct {
	Red    int `json:"red"`
	Yellow int `json:"yellow"`
}

type Incident

type Incident interface {
	Description() (string, error)
	// Notify() (error)
	Resolve() error
}

type Inventory

type Inventory struct {
	Cache     *scamp.ServiceCache
	Inventory InventoryType
}

func NewInventory

func NewInventory(cachePath string) (i *Inventory, err error)

func (*Inventory) Clone

func (i *Inventory) Clone() (i2 *Inventory)

func (*Inventory) Diff

func (old *Inventory) Diff(inv *Inventory) (diff map[string]InventoryDiffEntry)

func (*Inventory) Get

func (i *Inventory) Get(mangledName string) (val int, ok bool)

func (*Inventory) GetList

func (i *Inventory) GetList(mangledName string) (list []ServiceDesc, ok bool)

func (*Inventory) Reload

func (i *Inventory) Reload() (err error)

type InventoryDiffEntry

type InventoryDiffEntry struct {
	Missing []ServiceDesc `json:"missing"`
}

type InventoryType

type InventoryType map[string][]ServiceDesc

type PagerDutyEvent

type PagerDutyEvent struct {
	ServiceKey  string `json:"service_key"`
	Description string `json:"description"`
	EventType   string `json:"event_type"`

	Client      string                  `json:"client,omitempty"`       // optional
	ClientURL   string                  `json:"client_url,omitempty"`   // optional
	Contexts    []PagerDutyEventContext `json:"contexts,omitempty"`     // optional
	Details     interface{}             `json:"details,omitempty"`      // optional
	IncidentKey string                  `json:"incident_key,omitempty"` // optional
}

type PagerDutyEventContext

type PagerDutyEventContext struct {
	Href string `json:"href"`
	Type string `json:"type"`
}

type PagerdutyIncident

type PagerdutyIncident struct {
	Missing SectorHealth
}

func NewPagerdutyIncident

func NewPagerdutyIncident(pagerdutyKey string, missing SectorHealth) (yi *PagerdutyIncident)

func (*PagerdutyIncident) Description

func (pi *PagerdutyIncident) Description() (desc string, err error)

func (*PagerdutyIncident) Resolve

func (pi *PagerdutyIncident) Resolve() error

type SectorDigestMap

type SectorDigestMap map[string][]ServiceDesc

type SectorHealth

type SectorHealth map[string][]DegradedAction

func NewSectorHealth

func NewSectorHealth() SectorHealth

func (SectorHealth) IsDegraded

func (sh SectorHealth) IsDegraded() bool

func (SectorHealth) Red

func (sh SectorHealth) Red() (red SectorHealth)

func (SectorHealth) ToSectorHealthByInventory

func (sh SectorHealth) ToSectorHealthByInventory() (shi SectorHealthByInventory)

func (SectorHealth) Yellow

func (sh SectorHealth) Yellow() (red SectorHealth)

type SectorHealthByInventory

type SectorHealthByInventory map[string]SectorDigestMap

func NewSectorHealthByInventory

func NewSectorHealthByInventory() (shi SectorHealthByInventory)

func (SectorHealthByInventory) Add

func (shi SectorHealthByInventory) Add(sector, hostName, serviceName string, actionCount, actionInstances, actionTarget int)

type ServiceDesc

type ServiceDesc struct {
	ShortHostname string `json:"short_hostname"`
	ServiceName   string `json:"service_name"`
}

type SlackIncident

type SlackIncident struct {
	// contains filtered or unexported fields
}

func NewSlackIncident

func NewSlackIncident(expInv *ExpectedInventory, inv *Inventory) (yi *SlackIncident)

type SystemHealth

type SystemHealth struct {
	Missing []string `json:"missing"`
	Red     []string `json:"red"`
	Yellow  []string `json:"yellow"`
}

func NewSystemHealth

func NewSystemHealth() (sh *SystemHealth)

func (*SystemHealth) IsDegraded

func (sh *SystemHealth) IsDegraded() bool

func (*SystemHealth) MarkMissing

func (sh *SystemHealth) MarkMissing(mangledName string)

func (*SystemHealth) MarkRed

func (sh *SystemHealth) MarkRed(mangledName string)

func (*SystemHealth) MarkYellow

func (sh *SystemHealth) MarkYellow(mangledName string)