feat: M1 determinism fixes for gossip

Refs #1

- F1: canonical fingerprints (git identity by remote URL, log by
  basename, obsidian by relative vault path) so identical facts get
  identical ids across machines
- F2: node_id (persisted in settings) + Hybrid Logical Clock in all
  observation "when" columns; removed time.Now() fact-time fallbacks
- F3: stable TF-IDF tie-break sort (score desc, term asc)
- F4: observations carry (node_id, hcl) locator; dedup ordered by hcl
This commit is contained in:
2026-08-29 03:51:02 -07:00
parent f5766aa6d8
commit 2c0f8fa257
9 changed files with 162 additions and 30 deletions
+62 -6
View File
@@ -1,7 +1,9 @@
package db
import (
"crypto/rand"
"database/sql"
"encoding/hex"
"fmt"
"os"
"path/filepath"
@@ -10,13 +12,16 @@ import (
"sync"
"time"
"github.com/david/knox/internal/hlc"
_ "modernc.org/sqlite"
)
const goldenMinConfidence = 0.7
type KnoxDB struct {
db *sql.DB
db *sql.DB
nodeID string
clock *hlc.Clock
}
type Observation struct {
@@ -101,6 +106,9 @@ func Open(path string) (*KnoxDB, error) {
migrations := []string{
"ALTER TABLE threads ADD COLUMN provenance TEXT DEFAULT '{}'",
"ALTER TABLE entries ADD COLUMN created_at TEXT DEFAULT ''",
"ALTER TABLE observations ADD COLUMN node_id TEXT NOT NULL DEFAULT ''",
"ALTER TABLE observations ADD COLUMN hcl INTEGER",
"ALTER TABLE observations ADD COLUMN received_at TEXT",
}
for _, m := range migrations {
if _, err := db.Exec(m); err != nil && !strings.Contains(err.Error(), "duplicate column") {
@@ -108,9 +116,54 @@ func Open(path string) (*KnoxDB, error) {
}
}
return &KnoxDB{db: db}, nil
kdb := &KnoxDB{db: db, clock: hlc.New()}
// Node identity: generated once, persisted in settings. Stable across
// restarts so gossip cursors and locator IDs survive.
nodeID, err := kdb.loadNodeID()
if err != nil {
return nil, err
}
kdb.nodeID = nodeID
// Backfill: rows added before node_id existed belong to this node.
if _, err := db.Exec("UPDATE observations SET node_id=? WHERE node_id=''", nodeID); err != nil {
return nil, fmt.Errorf("backfill node_id: %w", err)
}
return kdb, nil
}
const nodeIDKey = "node_id"
// loadNodeID reads the persisted node identity, creating one if absent.
func (k *KnoxDB) loadNodeID() (string, error) {
var id string
err := k.db.QueryRow(`SELECT value FROM settings WHERE key=?`, nodeIDKey).Scan(&id)
if err == nil && id != "" {
return id, nil
}
if err != nil && err != sql.ErrNoRows {
return "", fmt.Errorf("read node_id: %w", err)
}
buf := make([]byte, 16)
if _, err := rand.Read(buf); err != nil {
return "", fmt.Errorf("generate node_id: %w", err)
}
id = hex.EncodeToString(buf)
if _, err := k.db.Exec(
`INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=?`,
nodeIDKey, id, id,
); err != nil {
return "", fmt.Errorf("persist node_id: %w", err)
}
return id, nil
}
// NodeID returns this node's stable identity.
func (k *KnoxDB) NodeID() string { return k.nodeID }
func (k *KnoxDB) Close() error { return k.db.Close() }
// RecordObservation appends an immutable observation and updates the entries cache.
@@ -123,7 +176,8 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
}
defer tx.Rollback()
now := time.Now().UTC().Format(time.RFC3339)
hclVal, hclWall := k.clock.Now()
now := hclWall.Format(time.RFC3339)
// Dedup: compare against the latest observation of the same fingerprint.
// For content sources (LineEnd > 0) the extent/summary IS the change signal —
@@ -136,7 +190,7 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
err = tx.QueryRow(
`SELECT id, COALESCE(title,''), COALESCE(summary,''), COALESCE(project,''),
COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0)
FROM observations WHERE fingerprint=? ORDER BY id DESC LIMIT 1`, o.Fingerprint,
FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, o.Fingerprint,
).Scan(&prevID, &pTitle, &pSummary, &pProject, &pCreated, &pStart, &pEnd)
switch err {
case nil:
@@ -154,10 +208,12 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
res, err := tx.Exec(
`INSERT INTO observations
(fingerprint, source_id, source_path, project, content_type, title, summary,
collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance,
node_id, hcl, received_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary,
now, o.CreatedAt, o.LineStart, o.LineEnd, o.Confidence, o.IngesterVersion, o.Trigger, o.Provenance,
k.nodeID, hclVal, time.Now().UTC().Format(time.RFC3339),
)
if err != nil {
return 0, false, fmt.Errorf("insert observation: %w", err)