feat: M1 determinism fixes for gossip
Refs #1 - F1: canonical fingerprints (git identity by remote URL, log by basename, obsidian by relative vault path) so identical facts get identical ids across machines - F2: node_id (persisted in settings) + Hybrid Logical Clock in all observation "when" columns; removed time.Now() fact-time fallbacks - F3: stable TF-IDF tie-break sort (score desc, term asc) - F4: observations carry (node_id, hcl) locator; dedup ordered by hcl
This commit is contained in:
+62
-6
@@ -1,7 +1,9 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,13 +12,16 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/david/knox/internal/hlc"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const goldenMinConfidence = 0.7
|
||||
|
||||
type KnoxDB struct {
|
||||
db *sql.DB
|
||||
db *sql.DB
|
||||
nodeID string
|
||||
clock *hlc.Clock
|
||||
}
|
||||
|
||||
type Observation struct {
|
||||
@@ -101,6 +106,9 @@ func Open(path string) (*KnoxDB, error) {
|
||||
migrations := []string{
|
||||
"ALTER TABLE threads ADD COLUMN provenance TEXT DEFAULT '{}'",
|
||||
"ALTER TABLE entries ADD COLUMN created_at TEXT DEFAULT ''",
|
||||
"ALTER TABLE observations ADD COLUMN node_id TEXT NOT NULL DEFAULT ''",
|
||||
"ALTER TABLE observations ADD COLUMN hcl INTEGER",
|
||||
"ALTER TABLE observations ADD COLUMN received_at TEXT",
|
||||
}
|
||||
for _, m := range migrations {
|
||||
if _, err := db.Exec(m); err != nil && !strings.Contains(err.Error(), "duplicate column") {
|
||||
@@ -108,9 +116,54 @@ func Open(path string) (*KnoxDB, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return &KnoxDB{db: db}, nil
|
||||
kdb := &KnoxDB{db: db, clock: hlc.New()}
|
||||
|
||||
// Node identity: generated once, persisted in settings. Stable across
|
||||
// restarts so gossip cursors and locator IDs survive.
|
||||
nodeID, err := kdb.loadNodeID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kdb.nodeID = nodeID
|
||||
|
||||
// Backfill: rows added before node_id existed belong to this node.
|
||||
if _, err := db.Exec("UPDATE observations SET node_id=? WHERE node_id=''", nodeID); err != nil {
|
||||
return nil, fmt.Errorf("backfill node_id: %w", err)
|
||||
}
|
||||
|
||||
return kdb, nil
|
||||
}
|
||||
|
||||
const nodeIDKey = "node_id"
|
||||
|
||||
// loadNodeID reads the persisted node identity, creating one if absent.
|
||||
func (k *KnoxDB) loadNodeID() (string, error) {
|
||||
var id string
|
||||
err := k.db.QueryRow(`SELECT value FROM settings WHERE key=?`, nodeIDKey).Scan(&id)
|
||||
if err == nil && id != "" {
|
||||
return id, nil
|
||||
}
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
return "", fmt.Errorf("read node_id: %w", err)
|
||||
}
|
||||
|
||||
buf := make([]byte, 16)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", fmt.Errorf("generate node_id: %w", err)
|
||||
}
|
||||
id = hex.EncodeToString(buf)
|
||||
if _, err := k.db.Exec(
|
||||
`INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=?`,
|
||||
nodeIDKey, id, id,
|
||||
); err != nil {
|
||||
return "", fmt.Errorf("persist node_id: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// NodeID returns this node's stable identity.
|
||||
func (k *KnoxDB) NodeID() string { return k.nodeID }
|
||||
|
||||
func (k *KnoxDB) Close() error { return k.db.Close() }
|
||||
|
||||
// RecordObservation appends an immutable observation and updates the entries cache.
|
||||
@@ -123,7 +176,8 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
hclVal, hclWall := k.clock.Now()
|
||||
now := hclWall.Format(time.RFC3339)
|
||||
|
||||
// Dedup: compare against the latest observation of the same fingerprint.
|
||||
// For content sources (LineEnd > 0) the extent/summary IS the change signal —
|
||||
@@ -136,7 +190,7 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
|
||||
err = tx.QueryRow(
|
||||
`SELECT id, COALESCE(title,''), COALESCE(summary,''), COALESCE(project,''),
|
||||
COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0)
|
||||
FROM observations WHERE fingerprint=? ORDER BY id DESC LIMIT 1`, o.Fingerprint,
|
||||
FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, o.Fingerprint,
|
||||
).Scan(&prevID, &pTitle, &pSummary, &pProject, &pCreated, &pStart, &pEnd)
|
||||
switch err {
|
||||
case nil:
|
||||
@@ -154,10 +208,12 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
|
||||
res, err := tx.Exec(
|
||||
`INSERT INTO observations
|
||||
(fingerprint, source_id, source_path, project, content_type, title, summary,
|
||||
collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance,
|
||||
node_id, hcl, received_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary,
|
||||
now, o.CreatedAt, o.LineStart, o.LineEnd, o.Confidence, o.IngesterVersion, o.Trigger, o.Provenance,
|
||||
k.nodeID, hclVal, time.Now().UTC().Format(time.RFC3339),
|
||||
)
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("insert observation: %w", err)
|
||||
|
||||
@@ -18,7 +18,10 @@ CREATE TABLE IF NOT EXISTS observations (
|
||||
confidence REAL DEFAULT 0.5,
|
||||
ingester_version TEXT DEFAULT '1',
|
||||
trigger TEXT,
|
||||
provenance TEXT DEFAULT '{}'
|
||||
provenance TEXT DEFAULT '{}',
|
||||
node_id TEXT NOT NULL DEFAULT '',
|
||||
hcl INTEGER,
|
||||
received_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_fingerprint ON observations(fingerprint);
|
||||
|
||||
Reference in New Issue
Block a user