fix: harden gossip, HLC restarts, watcher races, MCP args, pagination
- gossip: validate push batches (4 MiB / 1000-row caps); reject rows claiming the local node id (vector-poisoning), empty ids, negative HCLs - gossip: reconcile derived state after pulls (pulls only append to the observation log, so entry-count comparison could never trigger it) - hlc: seek clock from persisted MAX(hcl) at Open so a restart with a regressed wall clock cannot reissue values (locator/cursor safety) - db: serialize writers via BEGIN IMMEDIATE DSN, single conn per pool, and a per-KnoxDB mutex around RecordObservation's dedup - watch: atomic ticker guards (was a cross-goroutine data race), trailing-edge per-path debounce, recursive directory watches, rename re-ingest, remove cancels pending ingests - mcp: strict argument validation (no silent clamping), thread existence checks before writes, nil-safe golden-thread tool - cli: --page 0 no longer panics; query/recent pagination actually pages - tests: hlc SeekTo monotonicity, concurrent dedup race, push validation, batch caps, idempotency on observation counts
This commit is contained in:
+20
-1
@@ -22,6 +22,12 @@ type KnoxDB struct {
|
||||
db *sql.DB
|
||||
nodeID string
|
||||
clock *hlc.Clock
|
||||
|
||||
// writeMu serializes the check-then-insert dedup in RecordObservation within
|
||||
// this process. Cross-process serialization comes from _txlock=immediate (the
|
||||
// write lock is taken at BEGIN, before the dedup read) plus a single
|
||||
// connection per pool.
|
||||
writeMu sync.Mutex
|
||||
}
|
||||
|
||||
type Observation struct {
|
||||
@@ -93,10 +99,13 @@ func Open(path string) (*KnoxDB, error) {
|
||||
return nil, fmt.Errorf("create db dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_txlock=immediate")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open db: %w", err)
|
||||
}
|
||||
// One connection per process: WAL has a single writer; serializing on one
|
||||
// connection avoids pool contention surfacing as busy_timeout errors.
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
if _, err := db.Exec(Schema); err != nil {
|
||||
return nil, fmt.Errorf("init schema: %w", err)
|
||||
@@ -135,6 +144,13 @@ func Open(path string) (*KnoxDB, error) {
|
||||
if _, err := db.Exec("UPDATE observations SET hcl=id WHERE hcl IS NULL"); err != nil {
|
||||
return nil, fmt.Errorf("backfill hcl: %w", err)
|
||||
}
|
||||
// Resume this node's HLC from its persisted max: a restart with a regressed
|
||||
// wall clock must not reissue already-persisted values (see hlc.SeekTo).
|
||||
var maxHCL int64
|
||||
if err := db.QueryRow(`SELECT COALESCE(MAX(hcl), 0) FROM observations WHERE node_id=?`, nodeID).Scan(&maxHCL); err != nil {
|
||||
return nil, fmt.Errorf("seed hlc: %w", err)
|
||||
}
|
||||
kdb.clock.SeekTo(maxHCL)
|
||||
|
||||
// Locator uniqueness: (node_id, hcl) is the merge key for gossip; a given
|
||||
// node's HCL is strictly monotonic so this never throws a false conflict.
|
||||
@@ -204,6 +220,9 @@ func (k *KnoxDB) ObservationEntryEstimate() int {
|
||||
// Idempotent: if the latest observation for this fingerprint has an identical
|
||||
// content signature, nothing is recorded — re-ingesting unchanged content is a no-op.
|
||||
func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool, err error) {
|
||||
k.writeMu.Lock()
|
||||
defer k.writeMu.Unlock()
|
||||
|
||||
tx, err := k.db.Begin()
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("begin tx: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user