package db import ( "crypto/rand" "database/sql" "encoding/hex" "fmt" "os" "path/filepath" "regexp" "strings" "sync" "time" "github.com/david/knox/internal/hlc" _ "modernc.org/sqlite" ) const goldenMinConfidence = 0.7 type KnoxDB struct { db *sql.DB nodeID string clock *hlc.Clock } type Observation struct { ID int64 Fingerprint string SourceID string SourcePath string Project string ContentType string Title string Summary string CollectedAt string CreatedAt string LineStart int LineEnd int Confidence float64 IngesterVersion string Trigger string Count int // duplicates collapsed by ObservationsByFingerprint } type Entry struct { Fingerprint string SourceID string SourcePath string Project string ContentType string Title string Summary string FirstSeen string LastSeen string CreatedAt string RefCount int LastConfidence float64 } type SessionRow struct { SessionID string Project string Title string Status string AgentCount int StartedAt string EndedAt string LastSeen string Indexed bool } type ObservationRecord struct { Fingerprint string SourceID string SourcePath string Project string ContentType string Title string Summary string CreatedAt string LineStart int LineEnd int Confidence float64 IngesterVersion string Trigger string Provenance string } func Open(path string) (*KnoxDB, error) { dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0755); err != nil { return nil, fmt.Errorf("create db dir: %w", err) } db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") if err != nil { return nil, fmt.Errorf("open db: %w", err) } if _, err := db.Exec(Schema); err != nil { return nil, fmt.Errorf("init schema: %w", err) } // Migrations: safe ALTER TABLE for existing databases migrations := []string{ "ALTER TABLE threads ADD COLUMN provenance TEXT DEFAULT '{}'", "ALTER TABLE entries ADD COLUMN created_at TEXT DEFAULT ''", "ALTER TABLE observations ADD COLUMN node_id TEXT NOT NULL DEFAULT ''", "ALTER TABLE observations ADD COLUMN hcl INTEGER", "ALTER TABLE observations ADD COLUMN received_at TEXT", } for _, m := range migrations { if _, err := db.Exec(m); err != nil && !strings.Contains(err.Error(), "duplicate column") { return nil, fmt.Errorf("migration failed: %w", err) } } kdb := &KnoxDB{db: db, clock: hlc.New()} // Node identity: generated once, persisted in settings. Stable across // restarts so gossip cursors and locator IDs survive. nodeID, err := kdb.loadNodeID() if err != nil { return nil, err } kdb.nodeID = nodeID // Backfill: rows added before node_id existed belong to this node. if _, err := db.Exec("UPDATE observations SET node_id=? WHERE node_id=''", nodeID); err != nil { return nil, fmt.Errorf("backfill node_id: %w", err) } return kdb, nil } const nodeIDKey = "node_id" // loadNodeID reads the persisted node identity, creating one if absent. func (k *KnoxDB) loadNodeID() (string, error) { var id string err := k.db.QueryRow(`SELECT value FROM settings WHERE key=?`, nodeIDKey).Scan(&id) if err == nil && id != "" { return id, nil } if err != nil && err != sql.ErrNoRows { return "", fmt.Errorf("read node_id: %w", err) } buf := make([]byte, 16) if _, err := rand.Read(buf); err != nil { return "", fmt.Errorf("generate node_id: %w", err) } id = hex.EncodeToString(buf) if _, err := k.db.Exec( `INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=?`, nodeIDKey, id, id, ); err != nil { return "", fmt.Errorf("persist node_id: %w", err) } return id, nil } // NodeID returns this node's stable identity. func (k *KnoxDB) NodeID() string { return k.nodeID } func (k *KnoxDB) Close() error { return k.db.Close() } // RecordObservation appends an immutable observation and updates the entries cache. // Idempotent: if the latest observation for this fingerprint has an identical // content signature, nothing is recorded — re-ingesting unchanged content is a no-op. func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool, err error) { tx, err := k.db.Begin() if err != nil { return 0, false, fmt.Errorf("begin tx: %w", err) } defer tx.Rollback() hclVal, hclWall := k.clock.Now() now := hclWall.Format(time.RFC3339) // Dedup: compare against the latest observation of the same fingerprint. // For content sources (LineEnd > 0) the extent/summary IS the change signal — // created_at is just the file mtime, which can twitch without content change // (live log files). For extent-less sources (browser, gitea), created_at is // the only change signal (revisit, issue update) so it must match. var prevID int64 var pTitle, pSummary, pProject, pCreated string var pStart, pEnd int err = tx.QueryRow( `SELECT id, COALESCE(title,''), COALESCE(summary,''), COALESCE(project,''), COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0) FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, o.Fingerprint, ).Scan(&prevID, &pTitle, &pSummary, &pProject, &pCreated, &pStart, &pEnd) switch err { case nil: if pTitle == o.Title && pSummary == o.Summary && pProject == o.Project && pStart == o.LineStart && pEnd == o.LineEnd && (o.LineEnd > 0 || pCreated == o.CreatedAt) { return prevID, false, tx.Commit() // unchanged } case sql.ErrNoRows: // first observation of this fingerprint — proceed default: return 0, false, fmt.Errorf("check previous observation: %w", err) } res, err := tx.Exec( `INSERT INTO observations (fingerprint, source_id, source_path, project, content_type, title, summary, collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance, node_id, hcl, received_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary, now, o.CreatedAt, o.LineStart, o.LineEnd, o.Confidence, o.IngesterVersion, o.Trigger, o.Provenance, k.nodeID, hclVal, time.Now().UTC().Format(time.RFC3339), ) if err != nil { return 0, false, fmt.Errorf("insert observation: %w", err) } obsID, _ = res.LastInsertId() // Update materialized entries cache var existingFirstSeen string err = tx.QueryRow("SELECT first_seen FROM entries WHERE fingerprint=?", o.Fingerprint).Scan(&existingFirstSeen) isNew = false if err == sql.ErrNoRows { isNew = true _, err = tx.Exec( `INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`, o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary, now, now, coalesceCreatedAt(o.CreatedAt), o.Confidence, ) } else if err == nil { _, err = tx.Exec( `UPDATE entries SET last_seen=?, ref_count=ref_count+1, last_confidence=?, source_path=COALESCE(NULLIF(?,''), source_path), title=COALESCE(NULLIF(?,''), title), summary=COALESCE(NULLIF(?,''), summary), created_at=COALESCE(NULLIF(?,''), created_at) WHERE fingerprint=?`, now, o.Confidence, o.SourcePath, o.Title, o.Summary, o.CreatedAt, o.Fingerprint, ) } if err != nil { return 0, false, fmt.Errorf("update entries cache: %w", err) } // Auto-link to golden thread (only if relevant) if goldenID, keywords, bigrams := k.goldenThreadKeywords(tx); goldenID > 0 && len(keywords) > 0 { if o.Confidence >= goldenMinConfidence && isRelevant(o.Title, o.Summary, o.Project, keywords, bigrams) { res, err := tx.Exec( `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`, goldenID, obsID, ) linked := err == nil && res != nil if linked { if n, _ := res.RowsAffected(); n == 0 { linked = false } } tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, goldenID) // Temporal cross-reference: when a high-signal non-browser event links, // nearby browser history (±1h) becomes thread context too. if linked && o.SourceID != "browser-history" && o.CreatedAt != "" { k.linkTemporalNeighbors(tx, goldenID, o.CreatedAt, keywords, bigrams) } } } return obsID, isNew, tx.Commit() } // linkTemporalNeighbors links browser-history observations within ±1h of a // signal timestamp to the thread. A weak keyword check still applies so // unrelated browsing (lunch news) doesn't pollute the thread. func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt string, keywords []string, bigrams [][2]string) { rows, err := tx.Query( `SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,'') FROM observations o WHERE o.source_id='browser-history' AND o.created_at != '' AND ABS(strftime('%s', o.created_at) - strftime('%s', ?)) <= 3600 GROUP BY o.fingerprint`, createdAt, ) if err != nil { return } defer rows.Close() for rows.Next() { var obsID int64 var title, summary string if err := rows.Scan(&obsID, &title, &summary); err != nil { continue } if isRelevant(title, summary, "", keywords, bigrams) || weakRelevant(title, summary, keywords) { tx.Exec( `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'temporal')`, threadID, obsID, ) } } } // weakRelevant is a lower bar for temporal neighbors: any single keyword match. func weakRelevant(title, summary string, keywords []string) bool { text := strings.ToLower(title + " " + summary) for _, t := range wordBoundaryRE.FindAllString(text, -1) { for _, kw := range keywords { if t == kw { return true } } } return false } // FindEntry returns current entry from materialized cache. func (k *KnoxDB) FindEntry(fingerprint string) (*Entry, error) { row := k.db.QueryRow( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE fingerprint=?`, fingerprint, ) e := &Entry{} err := row.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType, &e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence) if err == sql.ErrNoRows { return nil, nil } return e, err } // ObservationsByFingerprint returns the observation history for a fingerprint, // collapsing exact duplicates (same summary/signal-time/extent) into one row // with a Count — re-observations of unchanged content are noise, not signal. func (k *KnoxDB) ObservationsByFingerprint(fp string, limit int) ([]Observation, error) { if limit <= 0 { limit = 50 } rows, err := k.db.Query( `SELECT MAX(id), fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), MAX(collected_at), COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0), confidence, ingester_version, COALESCE(trigger,''), COUNT(*) FROM observations WHERE fingerprint=? GROUP BY COALESCE(summary,''), COALESCE(created_at,''), COALESCE(line_end,0) ORDER BY MAX(collected_at) DESC LIMIT ?`, fp, limit, ) if err != nil { return nil, err } defer rows.Close() var obs []Observation for rows.Next() { var o Observation if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType, &o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd, &o.Confidence, &o.IngesterVersion, &o.Trigger, &o.Count); err != nil { return nil, err } obs = append(obs, o) } return obs, nil } // RebuildEntriesFromObservations drops and rebuilds the entries cache from observations. // coalesceCreatedAt prefers the observation's signal time, falling back to ingestion time. func coalesceCreatedAt(s string) string { if s != "" { return s } return time.Now().UTC().Format(time.RFC3339) } func (k *KnoxDB) RebuildEntriesFromObservations() error { tx, err := k.db.Begin() if err != nil { return err } defer tx.Rollback() if _, err := tx.Exec("DELETE FROM entries"); err != nil { return err } _, err = tx.Exec( `INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence) SELECT o.fingerprint, o.source_id, o.source_path, o.project, o.content_type, o.title, o.summary, MIN(o.collected_at), MAX(o.collected_at), COALESCE(NULLIF(MAX(o.created_at),''), MIN(o.collected_at)), COUNT(*), MAX(o.confidence) FROM observations o GROUP BY o.fingerprint`, ) if err != nil { return fmt.Errorf("rebuild entries: %w", err) } return tx.Commit() } // EntriesByProject returns entries from the materialized cache. func (k *KnoxDB) EntriesByProject(project string, limit int) ([]Entry, error) { if limit <= 0 { limit = 50 } rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE project=? ORDER BY last_seen DESC LIMIT ?`, project, limit, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } func (k *KnoxDB) RecentEntriesBySource(sourceID string, limit int) ([]Entry, error) { if limit <= 0 { limit = 10 } rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE source_id=? ORDER BY last_seen DESC LIMIT ?`, sourceID, limit, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } // EntriesBySource returns all entries for a source ordered by title (for // stable project-status listings like git repo snapshots). func (k *KnoxDB) EntriesBySource(sourceID string) ([]Entry, error) { rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE source_id=? ORDER BY title COLLATE NOCASE`, sourceID, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } func (k *KnoxDB) RecentEntries(limit int) ([]Entry, error) { if limit <= 0 { limit = 20 } rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries ORDER BY COALESCE(NULLIF(created_at,''), last_seen) DESC LIMIT ?`, limit, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } // escapeLike escapes LIKE wildcards so user input is matched literally. func escapeLike(s string) string { s = strings.ReplaceAll(s, `\`, `\\`) s = strings.ReplaceAll(s, `%`, `\%`) s = strings.ReplaceAll(s, `_`, `\_`) return s } func (k *KnoxDB) Search(query string, limit int) ([]Entry, error) { if limit <= 0 { limit = 20 } like := "%" + escapeLike(query) + "%" rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\' OR source_path LIKE ? ESCAPE '\' ORDER BY last_seen DESC LIMIT ?`, like, like, like, like, limit, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } // FindEntryByPrefix returns entries whose fingerprint starts with the given prefix. func (k *KnoxDB) FindEntryByPrefix(fp string, limit int) ([]Entry, error) { if limit <= 0 { limit = 5 } rows, err := k.db.Query( `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) FROM entries WHERE fingerprint LIKE ? || '%' LIMIT ?`, fp, limit, ) if err != nil { return nil, err } defer rows.Close() return scanEntries(rows) } func scanEntries(rows *sql.Rows) ([]Entry, error) { var entries []Entry for rows.Next() { var e Entry if err := rows.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType, &e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence); err != nil { return nil, err } entries = append(entries, e) } return entries, nil } // UpsertSession tracks session state. func (k *KnoxDB) UpsertSession(sessionID, project, title, status string) error { now := time.Now().UTC().Format(time.RFC3339) _, err := k.db.Exec( `INSERT INTO sessions (session_id, project, title, status, started_at, last_seen) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET title=COALESCE(NULLIF(?,''), title), status=?, last_seen=?, agent_count=agent_count+1`, sessionID, project, title, status, now, now, title, status, now, ) return err } func (k *KnoxDB) PendingSessions() ([]SessionRow, error) { rows, err := k.db.Query( `SELECT session_id, COALESCE(project,''), COALESCE(title,''), status, agent_count, COALESCE(started_at,''), COALESCE(ended_at,''), last_seen, indexed FROM sessions WHERE indexed=0 ORDER BY last_seen DESC`, ) if err != nil { return nil, err } defer rows.Close() var sessions []SessionRow for rows.Next() { var s SessionRow if err := rows.Scan(&s.SessionID, &s.Project, &s.Title, &s.Status, &s.AgentCount, &s.StartedAt, &s.EndedAt, &s.LastSeen, &s.Indexed); err != nil { return nil, err } sessions = append(sessions, s) } return sessions, nil } func (k *KnoxDB) MarkSessionIndexed(sessionID string) error { _, err := k.db.Exec("UPDATE sessions SET indexed=1 WHERE session_id=?", sessionID) return err } func (k *KnoxDB) Stats() (map[string]any, error) { stats := make(map[string]any) var v int var s string k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&v) stats["total_entries"] = v v = 0 k.db.QueryRow("SELECT COUNT(*) FROM entries WHERE last_seen > datetime('now', '-1 day')").Scan(&v) stats["entries_last_24h"] = v v = 0 k.db.QueryRow("SELECT COUNT(DISTINCT project) FROM entries").Scan(&v) stats["total_projects"] = v v = 0 k.db.QueryRow("SELECT COUNT(*) FROM observations").Scan(&v) stats["total_observations"] = v v = 0 k.db.QueryRow("SELECT COUNT(*) FROM observations WHERE collected_at > datetime('now', '-1 day')").Scan(&v) stats["observations_last_24h"] = v v = 0 k.db.QueryRow("SELECT COALESCE(MIN(collected_at),'') FROM observations").Scan(&s) stats["earliest_observation"] = s s = "" k.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&v) stats["total_sessions"] = v v = 0 k.db.QueryRow("SELECT COUNT(*) FROM sessions WHERE indexed=0").Scan(&v) stats["pending_reflections"] = v v = 0 goldenID, _ := k.GoldenThreadID() if goldenID > 0 { stats["golden_thread_id"] = goldenID t, _ := k.GetThread(goldenID) if t != nil { stats["golden_thread"] = t.Title } } return stats, nil } // ProvenanceChain returns the provenance link graph for a fingerprint. func (k *KnoxDB) ProvenanceChain(fp string, maxHops int) ([]struct { Fingerprint string Relation string HopDistance int }, error) { if maxHops <= 0 { maxHops = 3 } rows, err := k.db.Query( `SELECT ancestor_fp, relation, hop_distance FROM provenance_links WHERE descendant_fp=? AND hop_distance <= ? UNION SELECT descendant_fp, relation, hop_distance FROM provenance_links WHERE ancestor_fp=? AND hop_distance <= ? ORDER BY hop_distance`, fp, maxHops, fp, maxHops, ) if err != nil { return nil, err } defer rows.Close() var results []struct { Fingerprint string Relation string HopDistance int } for rows.Next() { var r struct { Fingerprint string Relation string HopDistance int } if err := rows.Scan(&r.Fingerprint, &r.Relation, &r.HopDistance); err != nil { return nil, err } results = append(results, r) } return results, nil } // ─── Golden Thread ─────────────────────────────────────────── func (k *KnoxDB) goldenThreadID(tx *sql.Tx) int64 { var id int64 err := tx.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id) if err != nil { return 0 } return id } func (k *KnoxDB) GoldenThreadID() (int64, error) { var id int64 err := k.db.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id) if err == sql.ErrNoRows { return 0, nil } return id, err } func (k *KnoxDB) SetGoldenThread(id int64) error { if id == 0 { _, err := k.db.Exec("DELETE FROM settings WHERE key='golden_thread_id'") return err } // Verify thread exists t, err := k.GetThread(id) if err != nil || t == nil { return fmt.Errorf("thread #%d not found", id) } _, err = k.db.Exec( "INSERT INTO settings (key, value) VALUES ('golden_thread_id', ?) ON CONFLICT(key) DO UPDATE SET value=?", id, id, ) return err } // kwCache caches DF-filtered golden-thread keywords, keyed by thread update time. var kwCache = struct { sync.Mutex threadID int64 updatedAt string keywords []string bigrams [][2]string }{threadID: -1} // maxDocFreq is the fraction of entries a keyword may match before it's // considered too generic to discriminate (e.g. "board"). const maxDocFreq = 0.02 func (k *KnoxDB) goldenThreadKeywords(tx *sql.Tx) (int64, []string, [][2]string) { var id int64 var title, motivation, tags, updatedAt string err := tx.QueryRow( `SELECT t.id, t.title, COALESCE(t.motivation,''), COALESCE(t.tags,'[]'), COALESCE(t.updated_at,'') FROM threads t JOIN settings s ON s.value = CAST(t.id AS TEXT) WHERE s.key='golden_thread_id'`, ).Scan(&id, &title, &motivation, &tags, &updatedAt) if err != nil { return 0, nil, nil } kwCache.Lock() if kwCache.threadID == id && kwCache.updatedAt == updatedAt { kws, bgs := kwCache.keywords, kwCache.bigrams kwCache.Unlock() return id, kws, bgs } kwCache.Unlock() raw := extractKeywords(title + " " + motivation + " " + tags) filtered := k.filterByDocFreq(tx, raw) // Keep only bigrams where both words survived DF filtering. survived := make(map[string]bool, len(filtered)) for _, kw := range filtered { survived[kw] = true } var bigrams [][2]string for _, bg := range extractBigrams(title + " " + motivation + " " + tags) { if survived[bg[0]] && survived[bg[1]] { bigrams = append(bigrams, bg) } } kwCache.Lock() kwCache.threadID, kwCache.updatedAt = id, updatedAt kwCache.keywords, kwCache.bigrams = filtered, bigrams kwCache.Unlock() return id, filtered, bigrams } // filterByDocFreq drops keywords that match more than maxDocFreq of all entries. // Rare keywords (klubhaus, doorbell) are discriminative; common ones (board, // current) just link noise. func (k *KnoxDB) filterByDocFreq(tx *sql.Tx, keywords []string) []string { var total int if err := tx.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 { return keywords } limit := int(float64(total) * maxDocFreq) if limit < 5 { limit = 5 } var out []string for _, kw := range keywords { var df int like := "%" + escapeLike(kw) + "%" err := tx.QueryRow( `SELECT COUNT(*) FROM entries WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`, like, like, like, ).Scan(&df) if err != nil || df <= limit { out = append(out, kw) } } return out } var wordRE = regexp.MustCompile(`[^a-z0-9]+`) // wordBoundaryRE extracts maximal alphanumeric runs. No \b anchors — Go treats // underscore as a word char, so \b would swallow snake_case tokens entirely // ("ALERT_klubhaus_topic" would yield nothing). var wordBoundaryRE = regexp.MustCompile(`[a-z0-9]+`) var stopWords = map[string]bool{ "with": true, "this": true, "that": true, "from": true, "they": true, "have": true, "were": true, "what": true, "when": true, "where": true, "which": true, "their": true, "them": true, "been": true, "also": true, "than": true, "into": true, "over": true, "such": true, "each": true, "about": true, "most": true, "some": true, "more": true, "other": true, "then": true, "will": true, "just": true, "like": true, "only": true, "very": true, "even": true, "much": true, "still": true, "well": true, "here": true, "there": true, "does": true, "done": true, "system": true, "page": true, "site": true, "user": true, "data": true, "file": true, "type": true, "home": true, "mode": true, "show": true, "name": true, "back": true, "open": true, } func ExtractKeywords(s string) []string { return extractKeywords(s) } func extractKeywords(s string) []string { s = strings.ToLower(s) words := wordRE.Split(s, -1) seen := make(map[string]bool) var keywords []string for _, w := range words { if len(w) > 3 && !seen[w] && !stopWords[w] { seen[w] = true keywords = append(keywords, w) } } return keywords } // extractBigrams returns ordered consecutive keyword pairs from the text, // preserving phrase structure ("multi board" from "multi-board"). func extractBigrams(s string) [][2]string { s = strings.ToLower(s) words := wordRE.Split(s, -1) var kept []string for _, w := range words { if len(w) > 3 && !stopWords[w] { kept = append(kept, w) } } seen := make(map[[2]string]bool) var out [][2]string for i := 0; i+1 < len(kept); i++ { pair := [2]string{kept[i], kept[i+1]} if !seen[pair] { seen[pair] = true out = append(out, pair) } } return out } // isRelevant scores phrase and keyword overlap: // - a bigram phrase match (in title or body) scores 3 — "multi board" is // precise, "board" alone is not // - a keyword in the title scores 2 // - a keyword in summary/project scores 1 // // Threshold is 3, so a single weak keyword never links by itself. func isRelevant(title, summary, project string, keywords []string, bigrams [][2]string) bool { titleTokens := wordBoundaryRE.FindAllString(strings.ToLower(title), -1) bodyTokens := wordBoundaryRE.FindAllString(strings.ToLower(summary+" "+project), -1) titleSet := make(map[string]bool, len(titleTokens)) for _, t := range titleTokens { titleSet[t] = true } bodySet := make(map[string]bool, len(bodyTokens)) for _, t := range bodyTokens { bodySet[t] = true } score := 0 for _, kw := range keywords { if titleSet[kw] { score += 2 } else if bodySet[kw] { score += 1 } } for _, bg := range bigrams { if containsBigram(titleTokens, bg) || containsBigram(bodyTokens, bg) { score += 3 } } return score >= 3 } func containsBigram(tokens []string, bg [2]string) bool { for i := 0; i+1 < len(tokens); i++ { if tokens[i] == bg[0] && tokens[i+1] == bg[1] { return true } } return false } // ─── Thread (Motivation/Intent) Operations ───────────────────── type Thread struct { ID int64 Title string Motivation string Status string Priority string CreatedAt string UpdatedAt string ResolvedAt string Tags string Provenance string EntryCount int } func ThreadFP(id int64) string { return fmt.Sprintf("thread:%d", id) } func (k *KnoxDB) CreateThread(title, motivation, priority, tags, provenance string) (int64, error) { res, err := k.db.Exec( `INSERT INTO threads (title, motivation, priority, tags, provenance) VALUES (?, ?, ?, ?, ?)`, title, motivation, priority, tags, provenance, ) if err != nil { return 0, err } return res.LastInsertId() } func (k *KnoxDB) ListThreads(status string) ([]Thread, error) { query := `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), COALESCE(t.provenance,'{}'), (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) FROM threads t` var args []any if status != "" { query += " WHERE t.status=?" args = append(args, status) } query += " ORDER BY t.updated_at DESC" rows, err := k.db.Query(query, args...) if err != nil { return nil, err } defer rows.Close() var threads []Thread for rows.Next() { var t Thread if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil { return nil, err } threads = append(threads, t) } return threads, nil } func (k *KnoxDB) GetThread(id int64) (*Thread, error) { row := k.db.QueryRow( `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), COALESCE(t.provenance,'{}'), (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) FROM threads t WHERE t.id=?`, id, ) t := &Thread{} err := row.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount) if err == sql.ErrNoRows { return nil, nil } return t, err } func (k *KnoxDB) CloseThread(id int64) error { _, err := k.db.Exec( `UPDATE threads SET status='resolved', resolved_at=datetime('now'), updated_at=datetime('now') WHERE id=?`, id, ) return err } // UpdateThread rewrites the editorial fields of a thread (used by the LLM // draft/polish pass to replace auto-generated titles/motivations). Empty // strings leave the corresponding field unchanged; omitting them all is a no-op. func (k *KnoxDB) UpdateThread(id int64, title, motivation, priority, tags string) (bool, error) { t, err := k.GetThread(id) if err != nil || t == nil { return false, fmt.Errorf("thread #%d not found", id) } if title == "" { title = t.Title } if motivation == "" { motivation = t.Motivation } if priority == "" { priority = t.Priority } if tags == "" { tags = t.Tags } changed := title != t.Title || motivation != t.Motivation || priority != t.Priority || tags != t.Tags if !changed { return false, nil } _, err = k.db.Exec( `UPDATE threads SET title=?, motivation=?, priority=?, tags=?, updated_at=datetime('now') WHERE id=?`, title, motivation, priority, tags, id, ) return true, err } func (k *KnoxDB) LinkObservationToThread(threadID, observationID int64, relevance string) error { _, err := k.db.Exec( `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, ?)`, threadID, observationID, relevance, ) if err != nil { return err } _, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID) return err } // AutoLinkThreadObservations resolves each entry fingerprint to its most recent // observation and links it to the thread ('auto_cluster' relevance). Idempotent: // INSERT OR IGNORE means re-linking an already-linked fingerprint is a no-op. // Returns the number of newly linked observations. func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []string) (int, error) { tx, err := k.db.Begin() if err != nil { return 0, err } defer tx.Rollback() linked := 0 for _, fp := range fingerprints { if fp == "" { continue } var obsID int64 if err := tx.QueryRow( `SELECT id FROM observations WHERE fingerprint=? ORDER BY id DESC LIMIT 1`, fp, ).Scan(&obsID); err != nil { continue } res, err := tx.Exec( `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto_cluster')`, threadID, obsID, ) if err != nil { continue } if n, _ := res.RowsAffected(); n > 0 { linked++ } } if linked > 0 { if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID); err != nil { return linked, err } } return linked, tx.Commit() } // ActiveThreadByKeyword returns the most recently updated active thread whose // title/motivation/tags contains the given keyword, or 0 if none exists. Used // to fold new clusters into an existing thread instead of spawning duplicates. func (k *KnoxDB) ActiveThreadByKeyword(keyword string) (int64, error) { if keyword == "" { return 0, nil } like := "%" + escapeLike(keyword) + "%" rows, err := k.db.Query( `SELECT id FROM threads WHERE status='active' AND (title LIKE ? ESCAPE '\' OR motivation LIKE ? ESCAPE '\' OR tags LIKE ? ESCAPE '\') ORDER BY updated_at DESC LIMIT 1`, like, like, like, ) if err != nil { return 0, err } defer rows.Close() if rows.Next() { var id int64 if err := rows.Scan(&id); err != nil { return 0, err } return id, nil } return 0, rows.Err() } // ThreadObservations returns linked observations, deduplicated by fingerprint // (re-observed entries appear once, with the latest observation's data). func (k *KnoxDB) ThreadObservations(threadID int64) ([]Observation, error) { rows, err := k.db.Query( `SELECT o.id, o.fingerprint, o.source_id, COALESCE(o.source_path,''), COALESCE(o.project,''), COALESCE(o.content_type,''), COALESCE(o.title,''), COALESCE(o.summary,''), o.collected_at, COALESCE(o.created_at,''), COALESCE(o.line_start,0), COALESCE(o.line_end,0), o.confidence, o.ingester_version, COALESCE(o.trigger,'') FROM observations o JOIN thread_observations to2 ON to2.observation_id = o.id WHERE to2.thread_id=? AND o.id IN ( SELECT MAX(to3.observation_id) FROM thread_observations to3 JOIN observations o2 ON o2.id = to3.observation_id WHERE to3.thread_id=? GROUP BY o2.fingerprint ) ORDER BY COALESCE(NULLIF(o.created_at,''), o.collected_at) DESC`, threadID, threadID, ) if err != nil { return nil, err } defer rows.Close() var obs []Observation for rows.Next() { var o Observation if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType, &o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd, &o.Confidence, &o.IngesterVersion, &o.Trigger); err != nil { return nil, err } obs = append(obs, o) } return obs, nil } func (k *KnoxDB) AddThreadNote(threadID int64, note string) (int64, error) { res, err := k.db.Exec( `INSERT INTO thread_notes (thread_id, note) VALUES (?, ?)`, threadID, note, ) if err != nil { return 0, err } _, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID) return res.LastInsertId() } func (k *KnoxDB) ThreadNotes(threadID int64) ([]struct { ID int64 Note string CreatedAt string }, error) { rows, err := k.db.Query( `SELECT id, note, created_at FROM thread_notes WHERE thread_id=? ORDER BY created_at`, threadID, ) if err != nil { return nil, err } defer rows.Close() var notes []struct { ID int64 Note string CreatedAt string } for rows.Next() { var n struct { ID int64 Note string CreatedAt string } if err := rows.Scan(&n.ID, &n.Note, &n.CreatedAt); err != nil { return nil, err } notes = append(notes, n) } return notes, nil } func (k *KnoxDB) SearchThreadsByMotivation(query string) ([]Thread, error) { like := "%" + escapeLike(query) + "%" rows, err := k.db.Query( `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), COALESCE(t.provenance,'{}'), (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) FROM threads t WHERE t.title LIKE ? ESCAPE '\' OR t.motivation LIKE ? ESCAPE '\' OR t.tags LIKE ? ESCAPE '\' ORDER BY t.updated_at DESC LIMIT 20`, like, like, like, ) if err != nil { return nil, err } defer rows.Close() var threads []Thread for rows.Next() { var t Thread if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil { return nil, err } threads = append(threads, t) } return threads, nil } // LinkEntryToThread links an entry to a thread via provenance_links. func (k *KnoxDB) LinkEntryToThread(threadID int64, entryFP, relation string) error { if entryFP == "" { return fmt.Errorf("entry fingerprint must not be empty") } tfp := ThreadFP(threadID) _, err := k.db.Exec( `INSERT OR IGNORE INTO provenance_links (descendant_fp, ancestor_fp, hop_distance, relation) VALUES (?, ?, 1, ?)`, entryFP, tfp, relation, ) return err } // ThreadRelevanceProfile returns the DF-filtered keywords and bigrams for any thread. func (k *KnoxDB) ThreadRelevanceProfile(threadID int64) ([]string, [][2]string, error) { var title, motivation, tags string err := k.db.QueryRow( `SELECT title, COALESCE(motivation,''), COALESCE(tags,'[]') FROM threads WHERE id=?`, threadID, ).Scan(&title, &motivation, &tags) if err != nil { return nil, nil, err } raw := extractKeywords(title + " " + motivation + " " + tags) var total int if err := k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 { return raw, nil, nil } limit := int(float64(total) * maxDocFreq) if limit < 5 { limit = 5 } survived := make(map[string]bool) var keywords []string for _, kw := range raw { var df int like := "%" + escapeLike(kw) + "%" if err := k.db.QueryRow( `SELECT COUNT(*) FROM entries WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`, like, like, like, ).Scan(&df); err != nil || df <= limit { keywords = append(keywords, kw) survived[kw] = true } } var bigrams [][2]string for _, bg := range extractBigrams(title + " " + motivation + " " + tags) { if survived[bg[0]] && survived[bg[1]] { bigrams = append(bigrams, bg) } } return keywords, bigrams, nil } // PruneThread removes machine-linked observations ('auto' and legacy 'regroom') // that fail the (stricter) relevance scoring, using the thread's DF-filtered // keyword profile. Temporal links and manual links (descriptive relevance // strings) are evidence of a different kind and are left intact. func (k *KnoxDB) PruneThread(threadID int64) (int, error) { keywords, bigrams, err := k.ThreadRelevanceProfile(threadID) if err != nil { return 0, err } if len(keywords) == 0 { return 0, nil } rows, err := k.db.Query( `SELECT to2.observation_id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'') FROM thread_observations to2 JOIN observations o ON o.id = to2.observation_id WHERE to2.thread_id = ? AND to2.relevance IN ('auto', 'regroom')`, threadID, ) if err != nil { return 0, err } var staleIDs []int64 for rows.Next() { var id int64 var title, summary, project string if err := rows.Scan(&id, &title, &summary, &project); err != nil { rows.Close() return 0, err } if !isRelevant(title, summary, project, keywords, bigrams) { staleIDs = append(staleIDs, id) } } rows.Close() if len(staleIDs) == 0 { return 0, nil } placeholders := strings.TrimSuffix(strings.Repeat("?,", len(staleIDs)), ",") args := make([]any, 0, len(staleIDs)+1) args = append(args, threadID) for _, id := range staleIDs { args = append(args, id) } res, err := k.db.Exec( fmt.Sprintf(`DELETE FROM thread_observations WHERE thread_id=? AND observation_id IN (%s)`, placeholders), args..., ) if err != nil { return 0, err } n, _ := res.RowsAffected() return int(n), nil } // RescanThread links existing observations that pass the thread's relevance // profile but were never linked — the inverse of PruneThread. One link row per // fingerprint (latest observation), relevance 'auto'. func (k *KnoxDB) RescanThread(threadID int64) (int, error) { keywords, bigrams, err := k.ThreadRelevanceProfile(threadID) if err != nil { return 0, err } if len(keywords) == 0 { return 0, nil } rows, err := k.db.Query( `SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'') FROM observations o WHERE o.id IN (SELECT MAX(id) FROM observations GROUP BY fingerprint) AND o.fingerprint NOT IN ( SELECT o2.fingerprint FROM thread_observations t JOIN observations o2 ON o2.id = t.observation_id WHERE t.thread_id = ? )`, threadID, ) if err != nil { return 0, err } var linkIDs []int64 for rows.Next() { var id int64 var title, summary, project string if err := rows.Scan(&id, &title, &summary, &project); err != nil { rows.Close() return 0, err } if isRelevant(title, summary, project, keywords, bigrams) { linkIDs = append(linkIDs, id) } } rows.Close() linked := 0 for _, id := range linkIDs { res, err := k.db.Exec( `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`, threadID, id, ) if err == nil { if n, _ := res.RowsAffected(); n > 0 { linked++ } } } return linked, nil } // ThreadProvenance returns the provenance graph for a thread. func (k *KnoxDB) ThreadProvenance(threadID int64) ([]struct { Fingerprint string Relation string HopDistance int }, error) { return k.ProvenanceChain(ThreadFP(threadID), 3) }