fix: harden gossip, HLC restarts, watcher races, MCP args, pagination

- gossip: validate push batches (4 MiB / 1000-row caps); reject rows claiming the local node id (vector-poisoning), empty ids, negative HCLs - gossip: reconcile derived state after pulls (pulls only append to the observation log, so entry-count comparison could never trigger it) - hlc: seek clock from persisted MAX(hcl) at Open so a restart with a regressed wall clock cannot reissue values (locator/cursor safety) - db: serialize writers via BEGIN IMMEDIATE DSN, single conn per pool, and a per-KnoxDB mutex around RecordObservation's dedup - watch: atomic ticker guards (was a cross-goroutine data race), trailing-edge per-path debounce, recursive directory watches, rename re-ingest, remove cancels pending ingests - mcp: strict argument validation (no silent clamping), thread existence checks before writes, nil-safe golden-thread tool - cli: --page 0 no longer panics; query/recent pagination actually pages - tests: hlc SeekTo monotonicity, concurrent dedup race, push validation, batch caps, idempotency on observation counts
2026-09-17 01:52:06 -07:00
parent 876d2aa45f
commit 6845975b7b
10 changed files with 733 additions and 116 deletions
@@ -0,0 +1,156 @@
+package db
+
+import (
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+func tmpDB(t *testing.T) *KnoxDB {
+	t.Helper()
+	kdb, err := Open(filepath.Join(t.TempDir(), "index.db"))
+	if err != nil {
+		t.Fatalf("open db: %v", err)
+	}
+	t.Cleanup(func() { kdb.Close() })
+	return kdb
+}
+
+func record(t *testing.T, k *KnoxDB, fp, title string) {
+	t.Helper()
+	_, _, err := k.RecordObservation(ObservationRecord{
+		Fingerprint:     fp,
+		SourceID:        "test",
+		SourcePath:      fp,
+		Project:         "itest",
+		ContentType:     "test",
+		Title:           title,
+		Summary:         "summary",
+		CreatedAt:       "2026-08-29T00:00:00Z",
+		LineEnd:         0,
+		Confidence:      0.9,
+		IngesterVersion: "test/v1",
+	})
+	if err != nil {
+		t.Errorf("record %s: %v", fp, err)
+	}
+}
+
+func obsCount(t *testing.T, k *KnoxDB) int {
+	t.Helper()
+	stats, err := k.Stats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	n, _ := stats["total_observations"].(int)
+	return n
+}
+
+// TestRecordObservationConcurrentDedup: N goroutines ingesting identical
+// content must produce exactly one observation row. This exercises the
+// check-then-insert dedup under the writeMu + BEGIN IMMEDIATE serialization.
+func TestRecordObservationConcurrentDedup(t *testing.T) {
+	k := tmpDB(t)
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			record(t, k, "fp:concurrent", "same title")
+		}()
+	}
+	wg.Wait()
+	if n := obsCount(t, k); n != 1 {
+		t.Fatalf("expected exactly 1 observation after concurrent identical ingests, got %d", n)
+	}
+}
+
+// TestRecordObservationDistinctFingerprints: different content must never be
+// deduped away (the constraint is per-fingerprint signature, not global).
+func TestRecordObservationDistinctFingerprints(t *testing.T) {
+	k := tmpDB(t)
+	record(t, k, "fp:a", "alpha")
+	record(t, k, "fp:b", "beta")
+	record(t, k, "fp:a", "alpha changed")
+	if n := obsCount(t, k); n != 3 {
+		t.Fatalf("expected 3 observations, got %d", n)
+	}
+}
+
+// TestPushObservationsValidation: forged/malformed rows are rejected without
+// error — self node_id (poisoning vector), empty node_id, negative HCL.
+func TestPushObservationsValidation(t *testing.T) {
+	k := tmpDB(t)
+	foreign := GossipObservation{
+		NodeID: "0123456789abcdef0123456789abcdef", HCL: 42,
+		Fingerprint: "fp:foreign", SourceID: "test", Title: "t", Summary: "s",
+		CollectedAt: "2026-08-29T00:00:00Z",
+	}
+	rows := []GossipObservation{
+		foreign,
+		{NodeID: k.NodeID(), HCL: 100, Fingerprint: "fp:self"}, // spoof poisoning attempt
+		{NodeID: "", HCL: 1, Fingerprint: "fp:empty"},
+		{NodeID: "other", HCL: -5, Fingerprint: "fp:neg"},
+	}
+	n, err := k.PushObservations(rows)
+	if err != nil {
+		t.Fatalf("push: %v", err)
+	}
+	if n != 1 {
+		t.Fatalf("expected exactly the one valid row inserted, got %d", n)
+	}
+	if got := obsCount(t, k); got != 1 {
+		t.Fatalf("expected 1 observation in the log, got %d", got)
+	}
+}
+
+// TestOpenReopenHCLMonotonicAcrossRestart: reopening a DB must resume the HLC
+// from its persisted max (clock seeding), keep the node identity, and order the
+// new observation above every previous one.
+func TestOpenReopenHCLMonotonicAcrossRestart(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "index.db")
+
+	k1, err := Open(path)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	record(t, k1, "fp:r1", "one")
+	record(t, k1, "fp:r2", "two")
+	record(t, k1, "fp:r3", "three")
+	nodeID1 := k1.NodeID()
+
+	maxBefore := int64(0)
+	rows, err := k1.ObservationsAfter(nodeID1, 0, 100)
+	if err != nil {
+		t.Fatalf("obs after: %v", err)
+	}
+	for _, r := range rows {
+		if r.HCL > maxBefore {
+			maxBefore = r.HCL
+		}
+	}
+	if err := k1.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+
+	k2, err := Open(path)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+	defer k2.Close()
+	if k2.NodeID() != nodeID1 {
+		t.Errorf("node id changed across reopen: %q -> %q", nodeID1, k2.NodeID())
+	}
+	record(t, k2, "fp:r4", "four")
+
+	after, err := k2.ObservationsAfter(nodeID1, maxBefore, 100)
+	if err != nil {
+		t.Fatalf("obs after (reopened): %v", err)
+	}
+	if len(after) != 1 {
+		t.Fatalf("expected exactly the new observation above the pre-restart max, got %d rows", len(after))
+	}
+	if after[0].Fingerprint != "fp:r4" {
+		t.Errorf("unexpected row above max: %s", after[0].Fingerprint)
+	}
+}