fix: harden gossip, HLC restarts, watcher races, MCP args, pagination

- gossip: validate push batches (4 MiB / 1000-row caps); reject rows
  claiming the local node id (vector-poisoning), empty ids, negative HCLs
- gossip: reconcile derived state after pulls (pulls only append to the
  observation log, so entry-count comparison could never trigger it)
- hlc: seek clock from persisted MAX(hcl) at Open so a restart with a
  regressed wall clock cannot reissue values (locator/cursor safety)
- db: serialize writers via BEGIN IMMEDIATE DSN, single conn per pool,
  and a per-KnoxDB mutex around RecordObservation's dedup
- watch: atomic ticker guards (was a cross-goroutine data race),
  trailing-edge per-path debounce, recursive directory watches,
  rename re-ingest, remove cancels pending ingests
- mcp: strict argument validation (no silent clamping), thread existence
  checks before writes, nil-safe golden-thread tool
- cli: --page 0 no longer panics; query/recent pagination actually pages
- tests: hlc SeekTo monotonicity, concurrent dedup race, push validation,
  batch caps, idempotency on observation counts
This commit is contained in:
2026-09-17 01:52:06 -07:00
parent 876d2aa45f
commit 6845975b7b
10 changed files with 733 additions and 116 deletions
+108 -4
View File
@@ -1,8 +1,12 @@
package watch
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/david/knox/internal/db"
@@ -228,16 +232,116 @@ func TestGossipIdempotent(t *testing.T) {
defer sb.Close()
Run(b, nil, []string{sa.URL})
before, err := b.EntryCount()
entryBefore, err := b.EntryCount()
if err != nil {
t.Fatal(err)
}
stats, err := b.Stats()
if err != nil {
t.Fatal(err)
}
obsBefore, _ := stats["total_observations"].(int)
Run(b, nil, []string{sa.URL})
after, err := b.EntryCount()
entryAfter, err := b.EntryCount()
if err != nil {
t.Fatal(err)
}
if before != after {
t.Errorf("second sweep changed entry count: %d -> %d", before, after)
if entryBefore != entryAfter {
t.Errorf("second sweep changed entry count: %d -> %d", entryBefore, entryAfter)
}
stats, err = b.Stats()
if err != nil {
t.Fatal(err)
}
obsAfter, _ := stats["total_observations"].(int)
if obsBefore != obsAfter {
t.Errorf("second sweep duplicated observations: %d -> %d", obsBefore, obsAfter)
}
}
// TestHandleBatchRejectsOversizedBatch: more than maxBatchRows in one POST
// must be refused up front, before any insert work.
func TestHandleBatchRejectsOversizedBatch(t *testing.T) {
b := tmpKnoxDB(t)
node := &Node{Kdb: b, Name: "B"}
sv := httptest.NewServer(node.Handler())
defer sv.Close()
rows := make([]db.GossipObservation, maxBatchRows+1)
for i := range rows {
rows[i] = db.GossipObservation{
NodeID: "0123456789abcdef0123456789abcdef", HCL: int64(i + 1),
Fingerprint: "fp:oversized", SourceID: "test", Title: "t", Summary: "s",
CollectedAt: "2026-08-29T00:00:00Z",
}
}
body, _ := json.Marshal(rows)
resp, err := http.Post(sv.URL+"/v1/obs/batch", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("post: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("oversized batch: want 400, got %d", resp.StatusCode)
}
}
// TestHandleBatchRejectsHugeBody: an oversized body (beyond the 4 MiB cap)
// must be refused even when the row count is small.
func TestHandleBatchRejectsHugeBody(t *testing.T) {
b := tmpKnoxDB(t)
node := &Node{Kdb: b, Name: "B"}
sv := httptest.NewServer(node.Handler())
defer sv.Close()
rows := []db.GossipObservation{{
NodeID: "0123456789abcdef0123456789abcdef", HCL: 1,
Fingerprint: "fp:huge", SourceID: "test", Title: "t",
Summary: strings.Repeat("x", 5<<20), // 5 MiB summary
CollectedAt: "2026-08-29T00:00:00Z",
}}
body, _ := json.Marshal(rows)
resp, err := http.Post(sv.URL+"/v1/obs/batch", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("post: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Errorf("huge body: want 400, got %d", resp.StatusCode)
}
}
// TestHandleBatchSkipsSelfRows: rows claiming the receiver's own node_id are
// dropped at the HTTP layer too (the poisoning vector), reported as conflicts.
func TestHandleBatchSkipsSelfRows(t *testing.T) {
b := tmpKnoxDB(t)
node := &Node{Kdb: b, Name: "B"}
sv := httptest.NewServer(node.Handler())
defer sv.Close()
rows := []db.GossipObservation{
{NodeID: b.NodeID(), HCL: 1, Fingerprint: "fp:self1", SourceID: "test", Title: "t", Summary: "s", CollectedAt: "2026-08-29T00:00:00Z"},
{NodeID: b.NodeID(), HCL: 2, Fingerprint: "fp:self2", SourceID: "test", Title: "t", Summary: "s", CollectedAt: "2026-08-29T00:00:00Z"},
}
body, _ := json.Marshal(rows)
resp, err := http.Post(sv.URL+"/v1/obs/batch", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("post: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("want 200, got %d", resp.StatusCode)
}
var out struct {
Accepted int `json:"accepted"`
Conflict int `json:"conflict"`
}
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
t.Fatalf("decode: %v", err)
}
if out.Accepted != 0 || out.Conflict != 2 {
t.Errorf("self rows: want accepted=0 conflict=2, got accepted=%d conflict=%d", out.Accepted, out.Conflict)
}
}