david/knox
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: Prometheus metrics endpoint for knox nodes

Browse Source 
Refs #2

- /metrics served on a dedicated port (KNOX_METRICS_ADDR, default
  localhost:8932) via prometheus/client_golang, with Go runtime +
  process collectors
- DB-derived gauges refreshed per scrape: observations by source,
  last-24h observations, entries, projects, sessions, pending
  reflections, threads by status, peers, observations by origin node,
  knowledge vector (max hcl per node)
- live gossip counters (pulls/pushes, observations pulled/pushed,
  errors) incremented during the anti-entropy sweep; Run accepts an
  optional metrics handle (nil for one-shot CLI)
- knox_node_info{node_id,name} for scrape identification
- internal/metrics package + db MetricsSnapshot; tests for snapshot,
  scrape output, and counter increments
This commit is contained in:
David Gwilliam
2026-08-29 06:06:49 -07:00
parent 25a7112d8a
commit 876d2aa45f
10 changed files with 494 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/watch/gossip.go
+32 -4
View File
@@ -19,14 +19,16 @@ import (
"time"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/metrics"
)
const defaultPort = "8931"
type Node struct {
Kdb *db.KnoxDB
Name string
Addr string // advertised base URL, e.g. http://192.168.1.20:8931
Kdb *db.KnoxDB
Name string
Addr string // advertised base URL, e.g. http://192.168.1.20:8931
Metrics *metrics.Metrics
}
// pingResponse is the anti-entropy summary returned by /v1/ping.
@@ -269,7 +271,9 @@ func (c *Client) Diff() (*DiffSummary, error) {
// with any one node reveals who else is in the swarm, and those nodes are then
// swept too. There is no relay of observations — only membership is shared; each
// node pulls/pushes directly with every other node it learns about.
func Run(kdb *db.KnoxDB, static []string) {
//
// m, when non-nil, receives gossip event counters (nil for one-shot CLI runs).
func Run(kdb *db.KnoxDB, m *metrics.Metrics, static []string) {
myID := kdb.NodeID()
// Seed the work queue with static config plus persisted discoveries.
@@ -292,6 +296,9 @@ func Run(kdb *db.KnoxDB, static []string) {
p, err := c.Ping()
if err != nil {
log.Printf("[gossip] ping %s: %v", addr, err)
if m != nil {
m.IncrementErrors()
}
continue
}
if p.NodeID == myID {
@@ -323,13 +330,22 @@ func Run(kdb *db.KnoxDB, static []string) {
n, err := c.Pull(remoteNode, localHCL, kdb)
if err != nil {
log.Printf("[gossip] pull %s@%s: %v", remoteNode, addr, err)
if m != nil {
m.IncrementErrors()
}
continue
}
pulled += n
}
}
if m != nil {
m.IncrementPull(pulled)
}
pushed, _ := c.Push(kdb, p.Vector)
if m != nil {
m.IncrementPush(pushed)
}
if err := kdb.UpsertPeer(p.NodeID, addr, p.Name, vectorMax(p.Vector)); err != nil {
log.Printf("[gossip] peer upsert: %v", err)
@@ -368,6 +384,18 @@ func ListenAddr() (addr string) {
return "localhost:" + defaultPort
}
const defaultMetricsPort = "8932"
// MetricsAddr returns the Prometheus scrape address (KNOX_METRICS_ADDR or
// default). It is a separate port from gossip so scraping never contends with
// the peer protocol.
func MetricsAddr() (addr string) {
if addr = os.Getenv("KNOX_METRICS_ADDR"); addr != "" {
return addr
}
return "localhost:" + defaultMetricsPort
}
// PeerAddrs returns the configured peer list (KNOX_PEERS, comma-separated).
func PeerAddrs() []string {
raw := os.Getenv("KNOX_PEERS")
internal/watch/gossip_test.go
+6 -6
View File
@@ -60,8 +60,8 @@ func TestGossipConvergence(t *testing.T) {
defer sb.Close()
// A pulls from B, then B pulls from A (bidirectional sweep).
Run(a, []string{sb.URL})
Run(b, []string{sa.URL})
Run(a, nil, []string{sb.URL})
Run(b, nil, []string{sa.URL})
av, err := a.KnowledgeVector()
if err != nil {
@@ -107,11 +107,11 @@ func TestGossipSwarmDiscovery(t *testing.T) {
defer sc.Close()
// A discovers B (A pings B) so A can advertise B to the swarm.
Run(a, []string{sb.URL})
Run(a, nil, []string{sb.URL})
// C only knows A. A single sweep should surface B (membership in ping)
// and pull B's observations directly.
Run(c, []string{sa.URL})
Run(c, nil, []string{sa.URL})
// C must know B and hold all three origin logs.
peers, err := c.ListPeers()
@@ -227,12 +227,12 @@ func TestGossipIdempotent(t *testing.T) {
sb := httptest.NewServer(nodeB.Handler())
defer sb.Close()
Run(b, []string{sa.URL})
Run(b, nil, []string{sa.URL})
before, err := b.EntryCount()
if err != nil {
t.Fatal(err)
}
Run(b, []string{sa.URL})
Run(b, nil, []string{sa.URL})
after, err := b.EntryCount()
if err != nil {
t.Fatal(err)
internal/watch/watch.go
+16 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
"github.com/david/knox/internal/metrics"
"github.com/fsnotify/fsnotify"
)
@@ -24,6 +25,7 @@ type Watcher struct {
vault string
debounce time.Duration
fileIngesters []ingest.Ingester
metrics *metrics.Metrics
}
func New(kdb *db.KnoxDB, dirs []string) *Watcher {
@@ -40,6 +42,7 @@ func New(kdb *db.KnoxDB, dirs []string) *Watcher {
ingest.NewLogIngester(),
ingest.NewSkillsIngester(),
},
metrics: metrics.New(kdb, "knox"),
}
}
@@ -53,7 +56,7 @@ func (w *Watcher) Start() error {
// Start the gossip server first so peers can reach us while the initial
// seed is still ingesting. Vault/dirs are logged after the seed below.
gossipAddr := ListenAddr()
node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr}
node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics}
srv := &http.Server{Addr: gossipAddr, Handler: node.Handler()}
go func() {
log.Printf("[knox] gossip listening on %s", gossipAddr)
@@ -61,6 +64,17 @@ func (w *Watcher) Start() error {
log.Printf("[knox] gossip server: %v", err)
}
}()
// Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR).
metricsAddr := MetricsAddr()
metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler()}
go func() {
log.Printf("[knox] metrics listening on %s", metricsAddr)
if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Printf("[knox] metrics server: %v", err)
}
}()
if peers := PeerAddrs(); len(peers) > 0 {
log.Printf("[knox] gossip peers: %v", peers)
}
@@ -447,7 +461,7 @@ func (w *Watcher) syncGossip() {
before = n
}
Run(w.knoxDB, peers)
Run(w.knoxDB, w.metrics, peers)
// If new observations arrived, reconcile to pick up entries/threads they
// imply (deterministic log → derived rebuild).