fix: derived-state determinism, error visibility, metrics naming, HTTP hardening
- rebuild: fold observations per fingerprint in a total order (hcl DESC, node_id DESC, id DESC) so two nodes with identical logs rebuild identical entries (was: arbitrary bare-column row, merge-order dependent) - threads: CreateThreadCluster uses INSERT OR IGNORE + existing-id fallback — concurrent auto-threaders converge instead of hitting UNIQUE - errors surfaced instead of swallowed: scanEntries returns rows.Err(), Stats() fails fast on query errors, AutoLinkThreadObservations / linkTemporalNeighbors / golden-thread linking propagate failures, AddThreadNote + LinkObservationToThread write under one tx, watch records session upsert failures - gossip client: push checks HTTP status and reports errors (a broken push direction no longer looks like a silent success); gossip diff gets a 10s timeout so a dead peer cannot hang the CLI - ingest: failed source sweeps (obsidian/browser/gitea) are logged, and -d's help text now states its file-only scope - watch --quiet: fatal errors go to stderr instead of io.Discard - main: cobra SilenceErrors/SilenceUsage (errors print once, usage is not dumped on runtime failures); knox mcp exits 0 on SIGINT/SIGTERM - metrics: drop _total suffix from gauges (knox_observations, knox_entries, knox_projects, knox_sessions, knox_peers, knox_threads); _total stays on counters per Prometheus convention - http: ReadHeaderTimeout + IdleTimeout on gossip, metrics, and web servers tests: concurrent cluster-create idempotency, HCL-order rebuild fold (both merge orders), push HTTP-error surfacing; full suite + -race pass, gofmt clean
This commit is contained in:
@@ -2,6 +2,7 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/david/knox/internal/db"
|
"github.com/david/knox/internal/db"
|
||||||
"github.com/david/knox/internal/watch"
|
"github.com/david/knox/internal/watch"
|
||||||
@@ -77,7 +78,7 @@ func NewGossipCmd(kdb *db.KnoxDB) *cobra.Command {
|
|||||||
// gossipDiff compares this node's observation log + auto-threads with a peer's
|
// gossipDiff compares this node's observation log + auto-threads with a peer's
|
||||||
// via /v1/diff. Never writes; it is the preview for what a sync would adopt.
|
// via /v1/diff. Never writes; it is the preview for what a sync would adopt.
|
||||||
func gossipDiff(kdb *db.KnoxDB, peerAddr string) error {
|
func gossipDiff(kdb *db.KnoxDB, peerAddr string) error {
|
||||||
c := &watch.Client{Addr: peerAddr}
|
c := &watch.Client{Addr: peerAddr, Timeout: 10 * time.Second}
|
||||||
remote, err := c.Diff()
|
remote, err := c.Diff()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -97,6 +97,8 @@ func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
|
|||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
record(r)
|
record(r)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.Printf("[knox] obsidian ingest failed: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
log.Printf("[knox] ingesting browser-history...")
|
log.Printf("[knox] ingesting browser-history...")
|
||||||
@@ -104,12 +106,16 @@ func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
|
|||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
record(r)
|
record(r)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.Printf("[knox] browser-history ingest failed: %v", err)
|
||||||
}
|
}
|
||||||
log.Printf("[knox] ingesting gitea...")
|
log.Printf("[knox] ingesting gitea...")
|
||||||
if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
|
if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
record(r)
|
record(r)
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
log.Printf("[knox] gitea ingest failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
|
fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package cmd
|
package cmd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
"github.com/david/knox/internal/db"
|
"github.com/david/knox/internal/db"
|
||||||
"github.com/david/knox/internal/watch"
|
"github.com/david/knox/internal/watch"
|
||||||
@@ -25,7 +27,10 @@ func NewWatchCmd(kdb *db.KnoxDB) *cobra.Command {
|
|||||||
log.Printf("[knox] starting watcher, scanning %d directories", len(dirs))
|
log.Printf("[knox] starting watcher, scanning %d directories", len(dirs))
|
||||||
w := watch.New(kdb, dirs)
|
w := watch.New(kdb, dirs)
|
||||||
if err := w.Start(); err != nil {
|
if err := w.Start(); err != nil {
|
||||||
log.Fatalf("watcher error: %v", err)
|
// Not log.Fatalf: --quiet redirects the logger to io.Discard,
|
||||||
|
// so a fatal exit must reach the user on stderr directly.
|
||||||
|
fmt.Fprintf(os.Stderr, "watcher error: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+144
-72
@@ -308,18 +308,21 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
|
|||||||
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`,
|
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`,
|
||||||
goldenID, obsID,
|
goldenID, obsID,
|
||||||
)
|
)
|
||||||
linked := err == nil && res != nil
|
if err != nil {
|
||||||
if linked {
|
return 0, false, fmt.Errorf("auto-link golden thread: %w", err)
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
}
|
||||||
linked = false
|
affected, _ := res.RowsAffected()
|
||||||
}
|
linked := affected > 0
|
||||||
|
if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, goldenID); err != nil {
|
||||||
|
return 0, false, fmt.Errorf("bump golden thread updated_at: %w", err)
|
||||||
}
|
}
|
||||||
tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, goldenID)
|
|
||||||
|
|
||||||
// Temporal cross-reference: when a high-signal non-browser event links,
|
// Temporal cross-reference: when a high-signal non-browser event links,
|
||||||
// nearby browser history (±1h) becomes thread context too.
|
// nearby browser history (±1h) becomes thread context too.
|
||||||
if linked && o.SourceID != "browser-history" && o.CreatedAt != "" {
|
if linked && o.SourceID != "browser-history" && o.CreatedAt != "" {
|
||||||
k.linkTemporalNeighbors(tx, goldenID, o.CreatedAt, keywords, bigrams)
|
if err := k.linkTemporalNeighbors(tx, goldenID, o.CreatedAt, keywords, bigrams); err != nil {
|
||||||
|
return 0, false, fmt.Errorf("link temporal neighbors: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -330,7 +333,7 @@ func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool
|
|||||||
// linkTemporalNeighbors links browser-history observations within ±1h of a
|
// linkTemporalNeighbors links browser-history observations within ±1h of a
|
||||||
// signal timestamp to the thread. A weak keyword check still applies so
|
// signal timestamp to the thread. A weak keyword check still applies so
|
||||||
// unrelated browsing (lunch news) doesn't pollute the thread.
|
// unrelated browsing (lunch news) doesn't pollute the thread.
|
||||||
func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt string, keywords []string, bigrams [][2]string) {
|
func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt string, keywords []string, bigrams [][2]string) error {
|
||||||
rows, err := tx.Query(
|
rows, err := tx.Query(
|
||||||
`SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,'')
|
`SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,'')
|
||||||
FROM observations o
|
FROM observations o
|
||||||
@@ -341,7 +344,7 @@ func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt str
|
|||||||
createdAt,
|
createdAt,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return err
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
|
|
||||||
@@ -349,15 +352,18 @@ func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt str
|
|||||||
var obsID int64
|
var obsID int64
|
||||||
var title, summary string
|
var title, summary string
|
||||||
if err := rows.Scan(&obsID, &title, &summary); err != nil {
|
if err := rows.Scan(&obsID, &title, &summary); err != nil {
|
||||||
continue
|
return err
|
||||||
}
|
}
|
||||||
if isRelevant(title, summary, "", keywords, bigrams) || weakRelevant(title, summary, keywords) {
|
if isRelevant(title, summary, "", keywords, bigrams) || weakRelevant(title, summary, keywords) {
|
||||||
tx.Exec(
|
if _, err := tx.Exec(
|
||||||
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'temporal')`,
|
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'temporal')`,
|
||||||
threadID, obsID,
|
threadID, obsID,
|
||||||
)
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// weakRelevant is a lower bar for temporal neighbors: any single keyword match.
|
// weakRelevant is a lower bar for temporal neighbors: any single keyword match.
|
||||||
@@ -447,23 +453,39 @@ func (k *KnoxDB) RebuildEntriesFromObservations() (before, after int, err error)
|
|||||||
if _, err := tx.Exec("DELETE FROM entries"); err != nil {
|
if _, err := tx.Exec("DELETE FROM entries"); err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
|
// Content fields come from the latest observation per fingerprint, folded in
|
||||||
|
// a total order (hcl DESC, node_id DESC, id DESC) — deterministic for any two
|
||||||
|
// nodes holding identical logs, per the gossip spec (bare GROUP BY columns
|
||||||
|
// would take an arbitrary row, merge-order dependent).
|
||||||
_, err = tx.Exec(
|
_, err = tx.Exec(
|
||||||
`INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence)
|
`INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence)
|
||||||
SELECT
|
SELECT
|
||||||
o.fingerprint,
|
latest.fingerprint,
|
||||||
o.source_id,
|
latest.source_id,
|
||||||
o.source_path,
|
latest.source_path,
|
||||||
o.project,
|
latest.project,
|
||||||
o.content_type,
|
latest.content_type,
|
||||||
o.title,
|
latest.title,
|
||||||
o.summary,
|
latest.summary,
|
||||||
MIN(o.collected_at),
|
agg.first_seen,
|
||||||
MAX(o.collected_at),
|
agg.last_seen,
|
||||||
COALESCE(NULLIF(MAX(o.created_at),''), MIN(o.collected_at)),
|
agg.created_at,
|
||||||
COUNT(*),
|
agg.ref_count,
|
||||||
MAX(o.confidence)
|
agg.max_confidence
|
||||||
FROM observations o
|
FROM (
|
||||||
GROUP BY o.fingerprint`,
|
SELECT *, ROW_NUMBER() OVER (PARTITION BY fingerprint ORDER BY hcl DESC, node_id DESC, id DESC) AS rn
|
||||||
|
FROM observations
|
||||||
|
) latest
|
||||||
|
JOIN (
|
||||||
|
SELECT fingerprint,
|
||||||
|
MIN(collected_at) AS first_seen,
|
||||||
|
MAX(collected_at) AS last_seen,
|
||||||
|
COALESCE(NULLIF(MAX(created_at),''), MIN(collected_at)) AS created_at,
|
||||||
|
COUNT(*) AS ref_count,
|
||||||
|
MAX(confidence) AS max_confidence
|
||||||
|
FROM observations GROUP BY fingerprint
|
||||||
|
) agg ON agg.fingerprint = latest.fingerprint
|
||||||
|
WHERE latest.rn = 1`,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("rebuild entries: %w", err)
|
return 0, 0, fmt.Errorf("rebuild entries: %w", err)
|
||||||
@@ -604,6 +626,9 @@ func scanEntries(rows *sql.Rows) ([]Entry, error) {
|
|||||||
}
|
}
|
||||||
entries = append(entries, e)
|
entries = append(entries, e)
|
||||||
}
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return entries, nil
|
return entries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -655,38 +680,50 @@ func (k *KnoxDB) MarkSessionIndexed(sessionID string) error {
|
|||||||
func (k *KnoxDB) Stats() (map[string]any, error) {
|
func (k *KnoxDB) Stats() (map[string]any, error) {
|
||||||
stats := make(map[string]any)
|
stats := make(map[string]any)
|
||||||
|
|
||||||
var v int
|
// A broken DB reports an error, not zeros: every aggregate is checked.
|
||||||
var s string
|
countInt := func(query string) (int, error) {
|
||||||
|
var v int
|
||||||
|
if err := k.db.QueryRow(query).Scan(&v); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
ints := []struct {
|
||||||
|
key string
|
||||||
|
query string
|
||||||
|
}{
|
||||||
|
{"total_entries", "SELECT COUNT(*) FROM entries"},
|
||||||
|
{"entries_last_24h", "SELECT COUNT(*) FROM entries WHERE last_seen > datetime('now', '-1 day')"},
|
||||||
|
{"total_projects", "SELECT COUNT(DISTINCT project) FROM entries"},
|
||||||
|
{"total_observations", "SELECT COUNT(*) FROM observations"},
|
||||||
|
{"observations_last_24h", "SELECT COUNT(*) FROM observations WHERE collected_at > datetime('now', '-1 day')"},
|
||||||
|
{"total_sessions", "SELECT COUNT(*) FROM sessions"},
|
||||||
|
{"pending_reflections", "SELECT COUNT(*) FROM sessions WHERE indexed=0"},
|
||||||
|
}
|
||||||
|
for _, it := range ints {
|
||||||
|
v, err := countInt(it.query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("stats %s: %w", it.key, err)
|
||||||
|
}
|
||||||
|
stats[it.key] = v
|
||||||
|
}
|
||||||
|
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&v)
|
var earliest string
|
||||||
stats["total_entries"] = v
|
if err := k.db.QueryRow("SELECT COALESCE(MIN(collected_at),'') FROM observations").Scan(&earliest); err != nil {
|
||||||
v = 0
|
return nil, fmt.Errorf("stats earliest_observation: %w", err)
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM entries WHERE last_seen > datetime('now', '-1 day')").Scan(&v)
|
}
|
||||||
stats["entries_last_24h"] = v
|
stats["earliest_observation"] = earliest
|
||||||
v = 0
|
|
||||||
k.db.QueryRow("SELECT COUNT(DISTINCT project) FROM entries").Scan(&v)
|
|
||||||
stats["total_projects"] = v
|
|
||||||
v = 0
|
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM observations").Scan(&v)
|
|
||||||
stats["total_observations"] = v
|
|
||||||
v = 0
|
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM observations WHERE collected_at > datetime('now', '-1 day')").Scan(&v)
|
|
||||||
stats["observations_last_24h"] = v
|
|
||||||
v = 0
|
|
||||||
k.db.QueryRow("SELECT COALESCE(MIN(collected_at),'') FROM observations").Scan(&s)
|
|
||||||
stats["earliest_observation"] = s
|
|
||||||
s = ""
|
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&v)
|
|
||||||
stats["total_sessions"] = v
|
|
||||||
v = 0
|
|
||||||
k.db.QueryRow("SELECT COUNT(*) FROM sessions WHERE indexed=0").Scan(&v)
|
|
||||||
stats["pending_reflections"] = v
|
|
||||||
v = 0
|
|
||||||
|
|
||||||
goldenID, _ := k.GoldenThreadID()
|
goldenID, err := k.GoldenThreadID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("stats golden_thread: %w", err)
|
||||||
|
}
|
||||||
if goldenID > 0 {
|
if goldenID > 0 {
|
||||||
stats["golden_thread_id"] = goldenID
|
stats["golden_thread_id"] = goldenID
|
||||||
t, _ := k.GetThread(goldenID)
|
t, err := k.GetThread(goldenID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("stats golden_thread: %w", err)
|
||||||
|
}
|
||||||
if t != nil {
|
if t != nil {
|
||||||
stats["golden_thread"] = t.Title
|
stats["golden_thread"] = t.Title
|
||||||
}
|
}
|
||||||
@@ -989,20 +1026,26 @@ func (k *KnoxDB) CreateThread(title, motivation, priority, tags, provenance stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (k *KnoxDB) CreateThreadCluster(title, motivation, priority, tags, provenance, clusterKey string) (int64, bool, error) {
|
func (k *KnoxDB) CreateThreadCluster(title, motivation, priority, tags, provenance, clusterKey string) (int64, bool, error) {
|
||||||
if clusterKey != "" {
|
// INSERT OR IGNORE makes creation atomic under the partial unique index:
|
||||||
if existing := k.ThreadByClusterKey(clusterKey); existing != 0 {
|
// concurrent creators (thread ticker + post-pull reconcile, or two gossip
|
||||||
return existing, false, nil
|
// nodes converging on the same cluster) lose the race into a no-op instead
|
||||||
}
|
// of a UNIQUE error, then pick up the winner's id.
|
||||||
}
|
|
||||||
res, err := k.db.Exec(
|
res, err := k.db.Exec(
|
||||||
`INSERT INTO threads (title, motivation, priority, tags, provenance, cluster_key) VALUES (?, ?, ?, ?, ?, ?)`,
|
`INSERT OR IGNORE INTO threads (title, motivation, priority, tags, provenance, cluster_key) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||||
title, motivation, priority, tags, provenance, clusterKey,
|
title, motivation, priority, tags, provenance, clusterKey,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, false, err
|
return 0, false, err
|
||||||
}
|
}
|
||||||
id, _ := res.LastInsertId()
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
return id, true, nil
|
id, _ := res.LastInsertId()
|
||||||
|
return id, true, nil
|
||||||
|
}
|
||||||
|
// Ignored insert: a thread with this cluster key already exists.
|
||||||
|
if existing := k.ThreadByClusterKey(clusterKey); existing != 0 {
|
||||||
|
return existing, false, nil
|
||||||
|
}
|
||||||
|
return 0, false, fmt.Errorf("insert ignored but no thread with cluster key %q", clusterKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ThreadByClusterKey returns the id of the thread auto-created for a given
|
// ThreadByClusterKey returns the id of the thread auto-created for a given
|
||||||
@@ -1107,15 +1150,21 @@ func (k *KnoxDB) UpdateThread(id int64, title, motivation, priority, tags string
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (k *KnoxDB) LinkObservationToThread(threadID, observationID int64, relevance string) error {
|
func (k *KnoxDB) LinkObservationToThread(threadID, observationID int64, relevance string) error {
|
||||||
_, err := k.db.Exec(
|
tx, err := k.db.Begin()
|
||||||
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, ?)`,
|
|
||||||
threadID, observationID, relevance,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID)
|
defer tx.Rollback()
|
||||||
return err
|
if _, err := tx.Exec(
|
||||||
|
`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, ?)`,
|
||||||
|
threadID, observationID, relevance,
|
||||||
|
); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
// AutoLinkThreadObservations resolves each entry fingerprint to its most recent
|
// AutoLinkThreadObservations resolves each entry fingerprint to its most recent
|
||||||
@@ -1130,6 +1179,7 @@ func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []strin
|
|||||||
defer tx.Rollback()
|
defer tx.Rollback()
|
||||||
|
|
||||||
linked := 0
|
linked := 0
|
||||||
|
var firstErr error
|
||||||
for _, fp := range fingerprints {
|
for _, fp := range fingerprints {
|
||||||
if fp == "" {
|
if fp == "" {
|
||||||
continue
|
continue
|
||||||
@@ -1138,6 +1188,11 @@ func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []strin
|
|||||||
if err := tx.QueryRow(
|
if err := tx.QueryRow(
|
||||||
`SELECT id FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, fp,
|
`SELECT id FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, fp,
|
||||||
).Scan(&obsID); err != nil {
|
).Scan(&obsID); err != nil {
|
||||||
|
// A fingerprint without observations is expected (provenance links);
|
||||||
|
// a real query failure is not — remember the first one.
|
||||||
|
if err != sql.ErrNoRows && firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
res, err := tx.Exec(
|
res, err := tx.Exec(
|
||||||
@@ -1145,6 +1200,9 @@ func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []strin
|
|||||||
threadID, obsID,
|
threadID, obsID,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if n, _ := res.RowsAffected(); n > 0 {
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
@@ -1157,7 +1215,10 @@ func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []strin
|
|||||||
return linked, err
|
return linked, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return linked, tx.Commit()
|
if err := tx.Commit(); err != nil {
|
||||||
|
return linked, err
|
||||||
|
}
|
||||||
|
return linked, firstErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// ActiveThreadByKeyword returns the most recently updated active thread whose
|
// ActiveThreadByKeyword returns the most recently updated active thread whose
|
||||||
@@ -1226,15 +1287,26 @@ func (k *KnoxDB) ThreadObservations(threadID int64) ([]Observation, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (k *KnoxDB) AddThreadNote(threadID int64, note string) (int64, error) {
|
func (k *KnoxDB) AddThreadNote(threadID int64, note string) (int64, error) {
|
||||||
res, err := k.db.Exec(
|
tx, err := k.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
res, err := tx.Exec(
|
||||||
`INSERT INTO thread_notes (thread_id, note) VALUES (?, ?)`,
|
`INSERT INTO thread_notes (thread_id, note) VALUES (?, ?)`,
|
||||||
threadID, note,
|
threadID, note,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
_, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID)
|
if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID); err != nil {
|
||||||
return res.LastInsertId()
|
return 0, err
|
||||||
|
}
|
||||||
|
id, err := res.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return id, tx.Commit()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k *KnoxDB) ThreadNotes(threadID int64) ([]struct {
|
func (k *KnoxDB) ThreadNotes(threadID int64) ([]struct {
|
||||||
|
|||||||
@@ -154,3 +154,89 @@ func TestOpenReopenHCLMonotonicAcrossRestart(t *testing.T) {
|
|||||||
t.Errorf("unexpected row above max: %s", after[0].Fingerprint)
|
t.Errorf("unexpected row above max: %s", after[0].Fingerprint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCreateThreadClusterConcurrentIdempotent: two concurrent creators of the
|
||||||
|
// same cluster key must converge on one thread (one created=true, one false)
|
||||||
|
// with no UNIQUE constraint error — the INSERT OR IGNORE path.
|
||||||
|
func TestCreateThreadClusterConcurrentIdempotent(t *testing.T) {
|
||||||
|
k := tmpDB(t)
|
||||||
|
const key = "cluster:race"
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
ids := make([]int64, 2)
|
||||||
|
created := make([]bool, 2)
|
||||||
|
errs := make([]error, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
ids[i], created[i], errs[i] = k.CreateThreadCluster("Race thread", "m", "medium", "[]", `{}`, key)
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
for i := range errs {
|
||||||
|
if errs[i] != nil {
|
||||||
|
t.Fatalf("creator %d: %v", i, errs[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ids[0] != ids[1] {
|
||||||
|
t.Errorf("concurrent creators got different ids: %d vs %d", ids[0], ids[1])
|
||||||
|
}
|
||||||
|
if created[0] == created[1] {
|
||||||
|
t.Errorf("exactly one creator should report created=true, got %v %v", created[0], created[1])
|
||||||
|
}
|
||||||
|
threads, err := k.ListThreads("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n := 0
|
||||||
|
for _, th := range threads {
|
||||||
|
if th.Title == "Race thread" {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n != 1 {
|
||||||
|
t.Errorf("want exactly 1 race thread, got %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRebuildEntriesDeterministicFold: two nodes holding the same observations
|
||||||
|
// in different merge orders must rebuild identical derived state — content
|
||||||
|
// fields come from the highest-HCL observation per fingerprint (spec 6.2),
|
||||||
|
// never from an arbitrary (merge-order-dependent) row.
|
||||||
|
func TestRebuildEntriesDeterministicFold(t *testing.T) {
|
||||||
|
const foreign = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||||
|
rows := []GossipObservation{
|
||||||
|
{NodeID: foreign, HCL: 10, Fingerprint: "fp:fold", SourceID: "test", Title: "old title", Summary: "old", CollectedAt: "2026-08-29T00:00:00Z", Confidence: 0.5},
|
||||||
|
{NodeID: foreign, HCL: 20, Fingerprint: "fp:fold", SourceID: "test", Title: "new title", Summary: "new", CollectedAt: "2026-08-29T01:00:00Z", Confidence: 0.9},
|
||||||
|
}
|
||||||
|
results := make([]Entry, 2)
|
||||||
|
for i, order := range [][]GossipObservation{{rows[0], rows[1]}, {rows[1], rows[0]}} {
|
||||||
|
k := tmpDB(t)
|
||||||
|
if _, err := k.PushObservations(order); err != nil {
|
||||||
|
t.Fatalf("push: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, err := k.RebuildEntriesFromObservations(); err != nil {
|
||||||
|
t.Fatalf("rebuild: %v", err)
|
||||||
|
}
|
||||||
|
e, err := k.FindEntry("fp:fold")
|
||||||
|
if err != nil || e == nil {
|
||||||
|
t.Fatalf("find entry: %v", err)
|
||||||
|
}
|
||||||
|
results[i] = *e
|
||||||
|
}
|
||||||
|
if results[0].Title != "new title" {
|
||||||
|
t.Errorf("order [old,new]: title = %q, want %q", results[0].Title, "new title")
|
||||||
|
}
|
||||||
|
if results[1].Title != "new title" {
|
||||||
|
t.Errorf("order [new,old]: title = %q, want %q", results[1].Title, "new title")
|
||||||
|
}
|
||||||
|
if results[0] != results[1] {
|
||||||
|
t.Errorf("derived state diverged between merge orders:\n%+v\n%+v", results[0], results[1])
|
||||||
|
}
|
||||||
|
if results[0].RefCount != 2 {
|
||||||
|
t.Errorf("ref_count = %d, want 2", results[0].RefCount)
|
||||||
|
}
|
||||||
|
if results[0].FirstSeen != "2026-08-29T00:00:00Z" || results[0].LastSeen != "2026-08-29T01:00:00Z" {
|
||||||
|
t.Errorf("first/last seen = %q/%q", results[0].FirstSeen, results[0].LastSeen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -59,14 +59,14 @@ func New(kdb *db.KnoxDB, nodeName string) *Metrics {
|
|||||||
m.obsPushedTotal = newCounter(reg, "knox_gossip_observations_pushed_total", "Observations sent to peers.")
|
m.obsPushedTotal = newCounter(reg, "knox_gossip_observations_pushed_total", "Observations sent to peers.")
|
||||||
m.errorsTotal = newCounter(reg, "knox_gossip_errors_total", "Gossip errors (ping/pull/push failures).")
|
m.errorsTotal = newCounter(reg, "knox_gossip_errors_total", "Gossip errors (ping/pull/push failures).")
|
||||||
|
|
||||||
m.observationsGauge = newGaugeVec(reg, "knox_observations_total", "Observation log size.", "source_id")
|
m.observationsGauge = newGaugeVec(reg, "knox_observations", "Observation log size.", "source_id")
|
||||||
m.observationsLast24h = newGauge(reg, "knox_observations_last_24h", "Observations collected in the last 24h.")
|
m.observationsLast24h = newGauge(reg, "knox_observations_last_24h", "Observations collected in the last 24h.")
|
||||||
m.entriesGauge = newGauge(reg, "knox_entries_total", "Materialized entry cache size.")
|
m.entriesGauge = newGauge(reg, "knox_entries", "Materialized entry cache size.")
|
||||||
m.projectsGauge = newGauge(reg, "knox_projects_total", "Distinct projects in the entry cache.")
|
m.projectsGauge = newGauge(reg, "knox_projects", "Distinct projects in the entry cache.")
|
||||||
m.sessionsGauge = newGauge(reg, "knox_sessions_total", "Sessions tracked.")
|
m.sessionsGauge = newGauge(reg, "knox_sessions", "Sessions tracked.")
|
||||||
m.pendingReflections = newGauge(reg, "knox_pending_reflections", "Sessions awaiting reflection.")
|
m.pendingReflections = newGauge(reg, "knox_pending_reflections", "Sessions awaiting reflection.")
|
||||||
m.peersGauge = newGauge(reg, "knox_peers_total", "Known peer nodes.")
|
m.peersGauge = newGauge(reg, "knox_peers", "Known peer nodes.")
|
||||||
m.threadsByStatus = newGaugeVec(reg, "knox_threads_total", "Threads by status.", "status")
|
m.threadsByStatus = newGaugeVec(reg, "knox_threads", "Threads by status.", "status")
|
||||||
m.byOriginNode = newGaugeVec(reg, "knox_observations_by_node", "Observations per originating node.", "node_id")
|
m.byOriginNode = newGaugeVec(reg, "knox_observations_by_node", "Observations per originating node.", "node_id")
|
||||||
m.knowledgeVector = newGaugeVec(reg, "knox_knowledge_max_hcl", "Highest HCL seen per originating node.", "node_id")
|
m.knowledgeVector = newGaugeVec(reg, "knox_knowledge_max_hcl", "Highest HCL seen per originating node.", "node_id")
|
||||||
|
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ func TestMetricsScrape(t *testing.T) {
|
|||||||
out := string(body)
|
out := string(body)
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
`knox_node_info{name="testnode"`,
|
`knox_node_info{name="testnode"`,
|
||||||
`knox_observations_total{source_id="git"} 2`,
|
`knox_observations{source_id="git"} 2`,
|
||||||
`knox_gossip_pulls_total 0`,
|
`knox_gossip_pulls_total 0`,
|
||||||
"go_goroutines",
|
"go_goroutines",
|
||||||
"process_cpu_seconds_total",
|
"process_cpu_seconds_total",
|
||||||
|
|||||||
@@ -42,7 +42,8 @@ func New(kdb *db.KnoxDB) *Server {
|
|||||||
|
|
||||||
func (s *Server) Serve(addr string) error {
|
func (s *Server) Serve(addr string) error {
|
||||||
log.Printf("[knox] web UI at http://%s", addr)
|
log.Printf("[knox] web UI at http://%s", addr)
|
||||||
return http.ListenAndServe(addr, s.mux)
|
srv := &http.Server{Addr: addr, Handler: s.mux, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second}
|
||||||
|
return srv.ListenAndServe()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Dashboard ───────────────────────────────────────────────
|
// ─── Dashboard ───────────────────────────────────────────────
|
||||||
|
|||||||
@@ -242,6 +242,9 @@ func (c *Client) Push(kdb *db.KnoxDB, peerVector map[string]int64) (int, error)
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode/100 != 2 {
|
||||||
|
return 0, fmt.Errorf("push to %s: %s", c.Addr, resp.Status)
|
||||||
|
}
|
||||||
var out struct {
|
var out struct {
|
||||||
Accepted int `json:"accepted"`
|
Accepted int `json:"accepted"`
|
||||||
}
|
}
|
||||||
@@ -354,7 +357,13 @@ func Run(kdb *db.KnoxDB, m *metrics.Metrics, static []string) int {
|
|||||||
m.IncrementPull(pulled)
|
m.IncrementPull(pulled)
|
||||||
}
|
}
|
||||||
|
|
||||||
pushed, _ := c.Push(kdb, p.Vector)
|
pushed, err := c.Push(kdb, p.Vector)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[gossip] push %s: %v", addr, err)
|
||||||
|
if m != nil {
|
||||||
|
m.IncrementErrors()
|
||||||
|
}
|
||||||
|
}
|
||||||
if m != nil {
|
if m != nil {
|
||||||
m.IncrementPush(pushed)
|
m.IncrementPush(pushed)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/david/knox/internal/db"
|
"github.com/david/knox/internal/db"
|
||||||
)
|
)
|
||||||
@@ -299,7 +300,7 @@ func TestHandleBatchRejectsHugeBody(t *testing.T) {
|
|||||||
rows := []db.GossipObservation{{
|
rows := []db.GossipObservation{{
|
||||||
NodeID: "0123456789abcdef0123456789abcdef", HCL: 1,
|
NodeID: "0123456789abcdef0123456789abcdef", HCL: 1,
|
||||||
Fingerprint: "fp:huge", SourceID: "test", Title: "t",
|
Fingerprint: "fp:huge", SourceID: "test", Title: "t",
|
||||||
Summary: strings.Repeat("x", 5<<20), // 5 MiB summary
|
Summary: strings.Repeat("x", 5<<20), // 5 MiB summary
|
||||||
CollectedAt: "2026-08-29T00:00:00Z",
|
CollectedAt: "2026-08-29T00:00:00Z",
|
||||||
}}
|
}}
|
||||||
body, _ := json.Marshal(rows)
|
body, _ := json.Marshal(rows)
|
||||||
@@ -345,3 +346,20 @@ func TestHandleBatchSkipsSelfRows(t *testing.T) {
|
|||||||
t.Errorf("self rows: want accepted=0 conflict=2, got accepted=%d conflict=%d", out.Accepted, out.Conflict)
|
t.Errorf("self rows: want accepted=0 conflict=2, got accepted=%d conflict=%d", out.Accepted, out.Conflict)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestClientPushSurfacesHTTPError: a non-2xx push response must surface as an
|
||||||
|
// error — silently treating it as accepted=0 would hide a broken sync direction.
|
||||||
|
func TestClientPushSurfacesHTTPError(t *testing.T) {
|
||||||
|
a := tmpKnoxDB(t)
|
||||||
|
seedObs(a, "AAA")
|
||||||
|
|
||||||
|
sv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "boom", http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer sv.Close()
|
||||||
|
|
||||||
|
c := &Client{Addr: sv.URL, Timeout: 5 * time.Second}
|
||||||
|
if _, err := c.Push(a, map[string]int64{}); err == nil {
|
||||||
|
t.Fatal("expected push error on HTTP 500, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ func (w *Watcher) Start() error {
|
|||||||
// seed is still ingesting. Vault/dirs are logged after the seed below.
|
// seed is still ingesting. Vault/dirs are logged after the seed below.
|
||||||
gossipAddr := ListenAddr()
|
gossipAddr := ListenAddr()
|
||||||
node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics}
|
node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics}
|
||||||
srv := &http.Server{Addr: gossipAddr, Handler: node.Handler()}
|
srv := &http.Server{Addr: gossipAddr, Handler: node.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second}
|
||||||
go func() {
|
go func() {
|
||||||
log.Printf("[knox] gossip listening on %s", gossipAddr)
|
log.Printf("[knox] gossip listening on %s", gossipAddr)
|
||||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
@@ -108,7 +108,7 @@ func (w *Watcher) Start() error {
|
|||||||
|
|
||||||
// Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR).
|
// Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR).
|
||||||
metricsAddr := MetricsAddr()
|
metricsAddr := MetricsAddr()
|
||||||
metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler()}
|
metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second}
|
||||||
go func() {
|
go func() {
|
||||||
log.Printf("[knox] metrics listening on %s", metricsAddr)
|
log.Printf("[knox] metrics listening on %s", metricsAddr)
|
||||||
if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
@@ -367,8 +367,9 @@ func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
|
|||||||
if result.SourceID == "opencode-session" {
|
if result.SourceID == "opencode-session" {
|
||||||
sessionID, _ := result.Provenance["session_id"].(string)
|
sessionID, _ := result.Provenance["session_id"].(string)
|
||||||
if sessionID != "" {
|
if sessionID != "" {
|
||||||
status := "active"
|
if err := w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, "active"); err != nil {
|
||||||
w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
|
log.Printf("[knox] session upsert error: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
@@ -75,6 +77,11 @@ and maintains a searchable index. Use 'knox watch' for daemon mode.`,
|
|||||||
s := knoxcmd.NewMCPServer(kdb)
|
s := knoxcmd.NewMCPServer(kdb)
|
||||||
log.Printf("[knox] starting MCP stdio server")
|
log.Printf("[knox] starting MCP stdio server")
|
||||||
if err := mcpServer.ServeStdio(s); err != nil {
|
if err := mcpServer.ServeStdio(s); err != nil {
|
||||||
|
// SIGINT/SIGTERM cancels the server context: clean shutdown,
|
||||||
|
// not a failure.
|
||||||
|
if errors.Is(err, context.Canceled) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
Reference in New Issue
Block a user