knox/internal/metrics/metrics.go

// Package metrics exposes Prometheus-format metrics for a knox node.
//
// Gauges are recomputed from the database on each scrape (cheap aggregates);
// gossip counters are in-memory and incremented as the daemon exchanges data
// with peers. The registry also gains the standard Go runtime and process
// collectors from prometheus/client_golang.
package metrics

import (
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/david/knox/internal/db"
)

// Metrics holds the gossip event counters (incremented by watch) and the
// scrape-time gauges derived from the database (refreshed on each scrape).
type Metrics struct {
	// Gossip counters (live).
	pullsTotal          prometheus.Counter
	pushesTotal         prometheus.Counter
	obsPulledTotal      prometheus.Counter
	obsPushedTotal      prometheus.Counter
	errorsTotal         prometheus.Counter

	// Snapshot gauges (updated per scrape).
	observationsGauge      *prometheus.GaugeVec
	entriesGauge           prometheus.Gauge
	projectsGauge          prometheus.Gauge
	sessionsGauge          prometheus.Gauge
	pendingReflections     prometheus.Gauge
	peersGauge             prometheus.Gauge
	threadsByStatus        *prometheus.GaugeVec
	byOriginNode           *prometheus.GaugeVec
	knowledgeVector        *prometheus.GaugeVec
	observationsLast24h    prometheus.Gauge

	registry *prometheus.Registry
	kdb      *db.KnoxDB
}

// New builds the metrics registry bound to a knowledge index.
func New(kdb *db.KnoxDB, nodeName string) *Metrics {
	reg := prometheus.NewRegistry()

	m := &Metrics{
		registry: reg,
		kdb:      kdb,
	}

	// Node identity aids scraping: which node produced this output.
	m.nodeInfo(kdb.NodeID(), nodeName)

	m.pullsTotal = newCounter(reg, "knox_gossip_pulls_total", "Peer pull round-trips completed.")
	m.pushesTotal = newCounter(reg, "knox_gossip_pushes_total", "Peer push round-trips completed.")
	m.obsPulledTotal = newCounter(reg, "knox_gossip_observations_pulled_total", "Observations received from peers.")
	m.obsPushedTotal = newCounter(reg, "knox_gossip_observations_pushed_total", "Observations sent to peers.")
	m.errorsTotal = newCounter(reg, "knox_gossip_errors_total", "Gossip errors (ping/pull/push failures).")

	m.observationsGauge = newGaugeVec(reg, "knox_observations_total", "Observation log size.", "source_id")
	m.observationsLast24h = newGauge(reg, "knox_observations_last_24h", "Observations collected in the last 24h.")
	m.entriesGauge = newGauge(reg, "knox_entries_total", "Materialized entry cache size.")
	m.projectsGauge = newGauge(reg, "knox_projects_total", "Distinct projects in the entry cache.")
	m.sessionsGauge = newGauge(reg, "knox_sessions_total", "Sessions tracked.")
	m.pendingReflections = newGauge(reg, "knox_pending_reflections", "Sessions awaiting reflection.")
	m.peersGauge = newGauge(reg, "knox_peers_total", "Known peer nodes.")
	m.threadsByStatus = newGaugeVec(reg, "knox_threads_total", "Threads by status.", "status")
	m.byOriginNode = newGaugeVec(reg, "knox_observations_by_node", "Observations per originating node.", "node_id")
	m.knowledgeVector = newGaugeVec(reg, "knox_knowledge_max_hcl", "Highest HCL seen per originating node.", "node_id")

	// Go runtime + process collectors come from the official library.
	reg.MustRegister(prometheus.NewGoCollector())
	reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))

	return m
}

func newCounter(reg *prometheus.Registry, name, help string) prometheus.Counter {
	c := prometheus.NewCounter(prometheus.CounterOpts{Name: name, Help: help})
	reg.MustRegister(c)
	return c
}

func newGauge(reg *prometheus.Registry, name, help string) prometheus.Gauge {
	g := prometheus.NewGauge(prometheus.GaugeOpts{Name: name, Help: help})
	reg.MustRegister(g)
	return g
}

func newGaugeVec(reg *prometheus.Registry, name, help string, labels ...string) *prometheus.GaugeVec {
	g := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels)
	reg.MustRegister(g)
	return g
}

func (m *Metrics) nodeInfo(nodeID, name string) {
	info := prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "knox_node_info",
		Help: "Node identity (always 1).",
		ConstLabels: prometheus.Labels{
			"node_id": nodeID,
			"name":    name,
		},
	})
	m.registry.MustRegister(info)
	info.Set(1)
}

// Capture refresh the DB-derived gauges from a fresh snapshot.
func (m *Metrics) Capture(s *db.MetricsSnapshot) {
	m.observationsGauge.Reset()
	for src, n := range s.BySource {
		m.observationsGauge.WithLabelValues(src).Set(float64(n))
	}
	m.observationsLast24h.Set(float64(s.ObservationsLast24h))
	m.entriesGauge.Set(float64(s.Entries))
	m.projectsGauge.Set(float64(s.Projects))
	m.sessionsGauge.Set(float64(s.Sessions))
	m.pendingReflections.Set(float64(s.PendingReflections))
	m.peersGauge.Set(float64(s.Peers))

	m.threadsByStatus.Reset()
	for st, n := range s.ThreadsByStatus {
		m.threadsByStatus.WithLabelValues(st).Set(float64(n))
	}

	m.byOriginNode.Reset()
	for nid, n := range s.ByOriginNode {
		m.byOriginNode.WithLabelValues(nid).Set(float64(n))
	}
	m.knowledgeVector.Reset()
	for nid, hcl := range s.KnowledgeVector {
		m.knowledgeVector.WithLabelValues(nid).Set(float64(hcl))
	}
}

// IncrementPull records a completed pull and its accepted observation count.
func (m *Metrics) IncrementPull(newObs int) {
	m.pullsTotal.Inc()
	m.obsPulledTotal.Add(float64(newObs))
}

// IncrementPush records a completed push and its accepted observation count.
func (m *Metrics) IncrementPush(newObs int) {
	m.pushesTotal.Inc()
	m.obsPushedTotal.Add(float64(newObs))
}

// IncrementErrors counts a failed gossip attempt.
func (m *Metrics) IncrementErrors() { m.errorsTotal.Inc() }

// Handler returns the /metrics scrape handler. On each scrape it refreshes
// DB-derived gauges before rendering (cost is a few cheap aggregates).
func (m *Metrics) Handler() http.Handler {
	h := promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if s, err := m.kdb.MetricsSnapshot(); err == nil {
			m.Capture(s)
		}
		h.ServeHTTP(w, r)
	})
}