knox/internal/db/db.go

package db

import (
	"crypto/rand"
	"database/sql"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/david/knox/internal/hlc"
	_ "modernc.org/sqlite"
)

const goldenMinConfidence = 0.7

type KnoxDB struct {
	db     *sql.DB
	nodeID string
	clock  *hlc.Clock

	// writeMu serializes the check-then-insert dedup in RecordObservation within
	// this process. Cross-process serialization comes from _txlock=immediate (the
	// write lock is taken at BEGIN, before the dedup read) plus a single
	// connection per pool.
	writeMu sync.Mutex
}

type Observation struct {
	ID              int64
	Fingerprint     string
	SourceID        string
	SourcePath      string
	Project         string
	ContentType     string
	Title           string
	Summary         string
	CollectedAt     string
	CreatedAt       string
	LineStart       int
	LineEnd         int
	Confidence      float64
	IngesterVersion string
	Trigger         string
	Count           int // duplicates collapsed by ObservationsByFingerprint
}

type Entry struct {
	Fingerprint    string
	SourceID       string
	SourcePath     string
	Project        string
	ContentType    string
	Title          string
	Summary        string
	FirstSeen      string
	LastSeen       string
	CreatedAt      string
	RefCount       int
	LastConfidence float64
}

type SessionRow struct {
	SessionID  string
	Project    string
	Title      string
	Status     string
	AgentCount int
	StartedAt  string
	EndedAt    string
	LastSeen   string
	Indexed    bool
}

type ObservationRecord struct {
	Fingerprint     string
	SourceID        string
	SourcePath      string
	Project         string
	ContentType     string
	Title           string
	Summary         string
	CreatedAt       string
	LineStart       int
	LineEnd         int
	Confidence      float64
	IngesterVersion string
	Trigger         string
	Provenance      string
}

func Open(path string) (*KnoxDB, error) {
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, fmt.Errorf("create db dir: %w", err)
	}

	db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_txlock=immediate")
	if err != nil {
		return nil, fmt.Errorf("open db: %w", err)
	}
	// One connection per process: WAL has a single writer; serializing on one
	// connection avoids pool contention surfacing as busy_timeout errors.
	db.SetMaxOpenConns(1)

	if _, err := db.Exec(Schema); err != nil {
		return nil, fmt.Errorf("init schema: %w", err)
	}

	// Migrations: safe ALTER TABLE for existing databases
	migrations := []string{
		"ALTER TABLE threads ADD COLUMN provenance TEXT DEFAULT '{}'",
		"ALTER TABLE entries ADD COLUMN created_at TEXT DEFAULT ''",
		"ALTER TABLE observations ADD COLUMN node_id TEXT NOT NULL DEFAULT ''",
		"ALTER TABLE observations ADD COLUMN hcl INTEGER",
		"ALTER TABLE observations ADD COLUMN received_at TEXT",
		"ALTER TABLE threads ADD COLUMN cluster_key TEXT",
	}
	for _, m := range migrations {
		if _, err := db.Exec(m); err != nil && !strings.Contains(err.Error(), "duplicate column") {
			return nil, fmt.Errorf("migration failed: %w", err)
		}
	}

	kdb := &KnoxDB{db: db, clock: hlc.New()}

	// Node identity: generated once, persisted in settings. Stable across
	// restarts so gossip cursors and locator IDs survive.
	nodeID, err := kdb.loadNodeID()
	if err != nil {
		return nil, err
	}
	kdb.nodeID = nodeID

	// Backfill: rows added before node_id existed belong to this node, and
	// their HCL is their local rowid (monotonic, so ordering is preserved).
	if _, err := db.Exec("UPDATE observations SET node_id=? WHERE node_id=''", nodeID); err != nil {
		return nil, fmt.Errorf("backfill node_id: %w", err)
	}
	if _, err := db.Exec("UPDATE observations SET hcl=id WHERE hcl IS NULL"); err != nil {
		return nil, fmt.Errorf("backfill hcl: %w", err)
	}
	// Resume this node's HLC from its persisted max: a restart with a regressed
	// wall clock must not reissue already-persisted values (see hlc.SeekTo).
	var maxHCL int64
	if err := db.QueryRow(`SELECT COALESCE(MAX(hcl), 0) FROM observations WHERE node_id=?`, nodeID).Scan(&maxHCL); err != nil {
		return nil, fmt.Errorf("seed hlc: %w", err)
	}
	kdb.clock.SeekTo(maxHCL)

	// Locator uniqueness: (node_id, hcl) is the merge key for gossip; a given
	// node's HCL is strictly monotonic so this never throws a false conflict.
	if _, err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_obs_locator ON observations(node_id, hcl)`); err != nil {
		return nil, fmt.Errorf("create locator index: %w", err)
	}

	// Thread idempotency index must come after the cluster_key migration (a
	// fresh DB has the column from Schema; an existing DB gets it above).
	if _, err := db.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_threads_cluster ON threads(cluster_key) WHERE cluster_key IS NOT NULL AND cluster_key != ''`); err != nil {
		return nil, fmt.Errorf("create cluster_key index: %w", err)
	}

	return kdb, nil
}

const nodeIDKey = "node_id"

// loadNodeID reads the persisted node identity, creating one if absent.
func (k *KnoxDB) loadNodeID() (string, error) {
	var id string
	err := k.db.QueryRow(`SELECT value FROM settings WHERE key=?`, nodeIDKey).Scan(&id)
	if err == nil && id != "" {
		return id, nil
	}
	if err != nil && err != sql.ErrNoRows {
		return "", fmt.Errorf("read node_id: %w", err)
	}

	buf := make([]byte, 16)
	if _, err := rand.Read(buf); err != nil {
		return "", fmt.Errorf("generate node_id: %w", err)
	}
	id = hex.EncodeToString(buf)
	if _, err := k.db.Exec(
		`INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=?`,
		nodeIDKey, id, id,
	); err != nil {
		return "", fmt.Errorf("persist node_id: %w", err)
	}
	return id, nil
}

// NodeID returns this node's stable identity.
func (k *KnoxDB) NodeID() string { return k.nodeID }

func (k *KnoxDB) Close() error { return k.db.Close() }

// EntryCount returns the number of entries in the materialized cache.
func (k *KnoxDB) EntryCount() (int, error) {
	var n int
	err := k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&n)
	return n, err
}

// ObservationEntryEstimate is the number of distinct fingerprints in the
// observation log — the ground-truth size reconcile's rebuild would produce.
func (k *KnoxDB) ObservationEntryEstimate() int {
	var n int
	if err := k.db.QueryRow("SELECT COUNT(DISTINCT fingerprint) FROM observations").Scan(&n); err != nil {
		return -1
	}
	return n
}

// RecordObservation appends an immutable observation and updates the entries cache.
// Idempotent: if the latest observation for this fingerprint has an identical
// content signature, nothing is recorded — re-ingesting unchanged content is a no-op.
func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool, err error) {
	k.writeMu.Lock()
	defer k.writeMu.Unlock()

	tx, err := k.db.Begin()
	if err != nil {
		return 0, false, fmt.Errorf("begin tx: %w", err)
	}
	defer tx.Rollback()

	hclVal, hclWall := k.clock.Now()
	now := hclWall.Format(time.RFC3339)

	// Dedup: compare against the latest observation of the same fingerprint.
	// For content sources (LineEnd > 0) the extent/summary IS the change signal —
	// created_at is just the file mtime, which can twitch without content change
	// (live log files). For extent-less sources (browser, gitea), created_at is
	// the only change signal (revisit, issue update) so it must match.
	var prevID int64
	var pTitle, pSummary, pProject, pCreated string
	var pStart, pEnd int
	err = tx.QueryRow(
		`SELECT id, COALESCE(title,''), COALESCE(summary,''), COALESCE(project,''),
		        COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0)
		 FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, o.Fingerprint,
	).Scan(&prevID, &pTitle, &pSummary, &pProject, &pCreated, &pStart, &pEnd)
	switch err {
	case nil:
		if pTitle == o.Title && pSummary == o.Summary && pProject == o.Project &&
			pStart == o.LineStart && pEnd == o.LineEnd &&
			(o.LineEnd > 0 || pCreated == o.CreatedAt) {
			return prevID, false, tx.Commit() // unchanged
		}
	case sql.ErrNoRows:
		// first observation of this fingerprint — proceed
	default:
		return 0, false, fmt.Errorf("check previous observation: %w", err)
	}

	res, err := tx.Exec(
		`INSERT INTO observations
		 (fingerprint, source_id, source_path, project, content_type, title, summary,
		  collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance,
		  node_id, hcl, received_at)
		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
		o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary,
		now, o.CreatedAt, o.LineStart, o.LineEnd, o.Confidence, o.IngesterVersion, o.Trigger, o.Provenance,
		k.nodeID, hclVal, time.Now().UTC().Format(time.RFC3339),
	)
	if err != nil {
		return 0, false, fmt.Errorf("insert observation: %w", err)
	}
	obsID, _ = res.LastInsertId()

	// Update materialized entries cache
	var existingFirstSeen string
	err = tx.QueryRow("SELECT first_seen FROM entries WHERE fingerprint=?", o.Fingerprint).Scan(&existingFirstSeen)

	isNew = false
	if err == sql.ErrNoRows {
		isNew = true
		_, err = tx.Exec(
			`INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence)
			 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`,
			o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary,
			now, now, coalesceCreatedAt(o.CreatedAt), o.Confidence,
		)
	} else if err == nil {
		_, err = tx.Exec(
			`UPDATE entries SET last_seen=?, ref_count=ref_count+1, last_confidence=?,
			 source_path=COALESCE(NULLIF(?,''), source_path),
			 title=COALESCE(NULLIF(?,''), title),
			 summary=COALESCE(NULLIF(?,''), summary),
			 created_at=COALESCE(NULLIF(?,''), created_at)
			 WHERE fingerprint=?`,
			now, o.Confidence, o.SourcePath, o.Title, o.Summary, o.CreatedAt, o.Fingerprint,
		)
	}
	if err != nil {
		return 0, false, fmt.Errorf("update entries cache: %w", err)
	}

	// Auto-link to golden thread (only if relevant)
	if goldenID, keywords, bigrams := k.goldenThreadKeywords(tx); goldenID > 0 && len(keywords) > 0 {
		if o.Confidence >= goldenMinConfidence && isRelevant(o.Title, o.Summary, o.Project, keywords, bigrams) {
			res, err := tx.Exec(
				`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`,
				goldenID, obsID,
			)
			linked := err == nil && res != nil
			if linked {
				if n, _ := res.RowsAffected(); n == 0 {
					linked = false
				}
			}
			tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, goldenID)

			// Temporal cross-reference: when a high-signal non-browser event links,
			// nearby browser history (±1h) becomes thread context too.
			if linked && o.SourceID != "browser-history" && o.CreatedAt != "" {
				k.linkTemporalNeighbors(tx, goldenID, o.CreatedAt, keywords, bigrams)
			}
		}
	}

	return obsID, isNew, tx.Commit()
}

// linkTemporalNeighbors links browser-history observations within ±1h of a
// signal timestamp to the thread. A weak keyword check still applies so
// unrelated browsing (lunch news) doesn't pollute the thread.
func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt string, keywords []string, bigrams [][2]string) {
	rows, err := tx.Query(
		`SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,'')
		 FROM observations o
		 WHERE o.source_id='browser-history'
		   AND o.created_at != ''
		   AND ABS(strftime('%s', o.created_at) - strftime('%s', ?)) <= 3600
		 GROUP BY o.fingerprint`,
		createdAt,
	)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		var obsID int64
		var title, summary string
		if err := rows.Scan(&obsID, &title, &summary); err != nil {
			continue
		}
		if isRelevant(title, summary, "", keywords, bigrams) || weakRelevant(title, summary, keywords) {
			tx.Exec(
				`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'temporal')`,
				threadID, obsID,
			)
		}
	}
}

// weakRelevant is a lower bar for temporal neighbors: any single keyword match.
func weakRelevant(title, summary string, keywords []string) bool {
	text := strings.ToLower(title + " " + summary)
	for _, t := range wordBoundaryRE.FindAllString(text, -1) {
		for _, kw := range keywords {
			if t == kw {
				return true
			}
		}
	}
	return false
}

// FindEntry returns current entry from materialized cache.
func (k *KnoxDB) FindEntry(fingerprint string) (*Entry, error) {
	row := k.db.QueryRow(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries WHERE fingerprint=?`, fingerprint,
	)
	e := &Entry{}
	err := row.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType,
		&e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence)
	if err == sql.ErrNoRows {
		return nil, nil
	}
	return e, err
}

// ObservationsByFingerprint returns the observation history for a fingerprint,
// collapsing exact duplicates (same summary/signal-time/extent) into one row
// with a Count — re-observations of unchanged content are noise, not signal.
func (k *KnoxDB) ObservationsByFingerprint(fp string, limit int) ([]Observation, error) {
	if limit <= 0 {
		limit = 50
	}
	rows, err := k.db.Query(
		`SELECT MAX(id), fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), MAX(collected_at), COALESCE(created_at,''),
		        COALESCE(line_start,0), COALESCE(line_end,0), confidence, ingester_version, COALESCE(trigger,''), COUNT(*)
		 FROM observations WHERE fingerprint=?
		 GROUP BY COALESCE(summary,''), COALESCE(created_at,''), COALESCE(line_end,0)
		 ORDER BY MAX(collected_at) DESC LIMIT ?`, fp, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var obs []Observation
	for rows.Next() {
		var o Observation
		if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType,
			&o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd,
			&o.Confidence, &o.IngesterVersion, &o.Trigger, &o.Count); err != nil {
			return nil, err
		}
		obs = append(obs, o)
	}
	return obs, nil
}

// RebuildEntriesFromObservations drops and rebuilds the entries cache from observations.
// coalesceCreatedAt prefers the observation's signal time, falling back to ingestion time.
func coalesceCreatedAt(s string) string {
	if s != "" {
		return s
	}
	return time.Now().UTC().Format(time.RFC3339)
}

func (k *KnoxDB) RebuildEntriesFromObservations() (before, after int, err error) {
	tx, err := k.db.Begin()
	if err != nil {
		return 0, 0, err
	}
	defer tx.Rollback()

	var b int
	if err := tx.QueryRow("SELECT COUNT(*) FROM entries").Scan(&b); err != nil {
		return 0, 0, err
	}

	if _, err := tx.Exec("DELETE FROM entries"); err != nil {
		return 0, 0, err
	}
	_, err = tx.Exec(
		`INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence)
		 SELECT
		   o.fingerprint,
		   o.source_id,
		   o.source_path,
		   o.project,
		   o.content_type,
		   o.title,
		   o.summary,
		   MIN(o.collected_at),
		   MAX(o.collected_at),
		   COALESCE(NULLIF(MAX(o.created_at),''), MIN(o.collected_at)),
		   COUNT(*),
		   MAX(o.confidence)
		 FROM observations o
		 GROUP BY o.fingerprint`,
	)
	if err != nil {
		return 0, 0, fmt.Errorf("rebuild entries: %w", err)
	}
	var a int
	if err := tx.QueryRow("SELECT COUNT(*) FROM entries").Scan(&a); err != nil {
		return 0, 0, err
	}
	if err := tx.Commit(); err != nil {
		return 0, 0, err
	}
	return b, a, nil
}

// EntriesByProject returns entries from the materialized cache.
func (k *KnoxDB) EntriesByProject(project string, limit int) ([]Entry, error) {
	if limit <= 0 {
		limit = 50
	}
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries WHERE project=? ORDER BY last_seen DESC LIMIT ?`, project, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

func (k *KnoxDB) RecentEntriesBySource(sourceID string, limit int) ([]Entry, error) {
	if limit <= 0 {
		limit = 10
	}
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries WHERE source_id=? ORDER BY last_seen DESC LIMIT ?`, sourceID, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

// EntriesBySource returns all entries for a source ordered by title (for
// stable project-status listings like git repo snapshots).
func (k *KnoxDB) EntriesBySource(sourceID string) ([]Entry, error) {
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries WHERE source_id=? ORDER BY title COLLATE NOCASE`, sourceID,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

func (k *KnoxDB) RecentEntries(limit int) ([]Entry, error) {
	if limit <= 0 {
		limit = 20
	}
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries ORDER BY COALESCE(NULLIF(created_at,''), last_seen) DESC LIMIT ?`, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

// escapeLike escapes LIKE wildcards so user input is matched literally.
func escapeLike(s string) string {
	s = strings.ReplaceAll(s, `\`, `\\`)
	s = strings.ReplaceAll(s, `%`, `\%`)
	s = strings.ReplaceAll(s, `_`, `\_`)
	return s
}

func (k *KnoxDB) Search(query string, limit int) ([]Entry, error) {
	if limit <= 0 {
		limit = 20
	}
	like := "%" + escapeLike(query) + "%"
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries
		 WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\'
		    OR project LIKE ? ESCAPE '\' OR source_path LIKE ? ESCAPE '\'
		 ORDER BY last_seen DESC LIMIT ?`,
		like, like, like, like, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

// FindEntryByPrefix returns entries whose fingerprint starts with the given prefix.
func (k *KnoxDB) FindEntryByPrefix(fp string, limit int) ([]Entry, error) {
	if limit <= 0 {
		limit = 5
	}
	rows, err := k.db.Query(
		`SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''),
		        COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''),
		        COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5)
		 FROM entries WHERE fingerprint LIKE ? || '%' LIMIT ?`, fp, limit,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	return scanEntries(rows)
}

func scanEntries(rows *sql.Rows) ([]Entry, error) {
	var entries []Entry
	for rows.Next() {
		var e Entry
		if err := rows.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType,
			&e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence); err != nil {
			return nil, err
		}
		entries = append(entries, e)
	}
	return entries, nil
}

// UpsertSession tracks session state.
func (k *KnoxDB) UpsertSession(sessionID, project, title, status string) error {
	now := time.Now().UTC().Format(time.RFC3339)
	_, err := k.db.Exec(
		`INSERT INTO sessions (session_id, project, title, status, started_at, last_seen)
		 VALUES (?, ?, ?, ?, ?, ?)
		 ON CONFLICT(session_id) DO UPDATE SET
		   title=COALESCE(NULLIF(?,''), title),
		   status=?,
		   last_seen=?,
		   agent_count=agent_count+1`,
		sessionID, project, title, status, now, now,
		title, status, now,
	)
	return err
}

func (k *KnoxDB) PendingSessions() ([]SessionRow, error) {
	rows, err := k.db.Query(
		`SELECT session_id, COALESCE(project,''), COALESCE(title,''), status, agent_count,
		        COALESCE(started_at,''), COALESCE(ended_at,''), last_seen, indexed
		 FROM sessions WHERE indexed=0 ORDER BY last_seen DESC`,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var sessions []SessionRow
	for rows.Next() {
		var s SessionRow
		if err := rows.Scan(&s.SessionID, &s.Project, &s.Title, &s.Status, &s.AgentCount,
			&s.StartedAt, &s.EndedAt, &s.LastSeen, &s.Indexed); err != nil {
			return nil, err
		}
		sessions = append(sessions, s)
	}
	return sessions, nil
}

func (k *KnoxDB) MarkSessionIndexed(sessionID string) error {
	_, err := k.db.Exec("UPDATE sessions SET indexed=1 WHERE session_id=?", sessionID)
	return err
}

func (k *KnoxDB) Stats() (map[string]any, error) {
	stats := make(map[string]any)

	var v int
	var s string

	k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&v)
	stats["total_entries"] = v
	v = 0
	k.db.QueryRow("SELECT COUNT(*) FROM entries WHERE last_seen > datetime('now', '-1 day')").Scan(&v)
	stats["entries_last_24h"] = v
	v = 0
	k.db.QueryRow("SELECT COUNT(DISTINCT project) FROM entries").Scan(&v)
	stats["total_projects"] = v
	v = 0
	k.db.QueryRow("SELECT COUNT(*) FROM observations").Scan(&v)
	stats["total_observations"] = v
	v = 0
	k.db.QueryRow("SELECT COUNT(*) FROM observations WHERE collected_at > datetime('now', '-1 day')").Scan(&v)
	stats["observations_last_24h"] = v
	v = 0
	k.db.QueryRow("SELECT COALESCE(MIN(collected_at),'') FROM observations").Scan(&s)
	stats["earliest_observation"] = s
	s = ""
	k.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&v)
	stats["total_sessions"] = v
	v = 0
	k.db.QueryRow("SELECT COUNT(*) FROM sessions WHERE indexed=0").Scan(&v)
	stats["pending_reflections"] = v
	v = 0

	goldenID, _ := k.GoldenThreadID()
	if goldenID > 0 {
		stats["golden_thread_id"] = goldenID
		t, _ := k.GetThread(goldenID)
		if t != nil {
			stats["golden_thread"] = t.Title
		}
	}

	return stats, nil
}

// ProvenanceChain returns the provenance link graph for a fingerprint.
func (k *KnoxDB) ProvenanceChain(fp string, maxHops int) ([]struct {
	Fingerprint string
	Relation    string
	HopDistance int
}, error) {
	if maxHops <= 0 {
		maxHops = 3
	}
	rows, err := k.db.Query(
		`SELECT ancestor_fp, relation, hop_distance FROM provenance_links
		 WHERE descendant_fp=? AND hop_distance <= ?
		 UNION
		 SELECT descendant_fp, relation, hop_distance FROM provenance_links
		 WHERE ancestor_fp=? AND hop_distance <= ?
		 ORDER BY hop_distance`, fp, maxHops, fp, maxHops,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var results []struct {
		Fingerprint string
		Relation    string
		HopDistance int
	}
	for rows.Next() {
		var r struct {
			Fingerprint string
			Relation    string
			HopDistance int
		}
		if err := rows.Scan(&r.Fingerprint, &r.Relation, &r.HopDistance); err != nil {
			return nil, err
		}
		results = append(results, r)
	}
	return results, nil
}

// ─── Golden Thread ───────────────────────────────────────────

func (k *KnoxDB) goldenThreadID(tx *sql.Tx) int64 {
	var id int64
	err := tx.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id)
	if err != nil {
		return 0
	}
	return id
}

func (k *KnoxDB) GoldenThreadID() (int64, error) {
	var id int64
	err := k.db.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id)
	if err == sql.ErrNoRows {
		return 0, nil
	}
	return id, err
}

func (k *KnoxDB) SetGoldenThread(id int64) error {
	if id == 0 {
		_, err := k.db.Exec("DELETE FROM settings WHERE key='golden_thread_id'")
		return err
	}
	// Verify thread exists
	t, err := k.GetThread(id)
	if err != nil || t == nil {
		return fmt.Errorf("thread #%d not found", id)
	}
	_, err = k.db.Exec(
		"INSERT INTO settings (key, value) VALUES ('golden_thread_id', ?) ON CONFLICT(key) DO UPDATE SET value=?",
		id, id,
	)
	return err
}

// kwCache caches DF-filtered golden-thread keywords, keyed by thread update time.
var kwCache = struct {
	sync.Mutex
	threadID  int64
	updatedAt string
	keywords  []string
	bigrams   [][2]string
}{threadID: -1}

// maxDocFreq is the fraction of entries a keyword may match before it's
// considered too generic to discriminate (e.g. "board").
const maxDocFreq = 0.02

func (k *KnoxDB) goldenThreadKeywords(tx *sql.Tx) (int64, []string, [][2]string) {
	var id int64
	var title, motivation, tags, updatedAt string
	err := tx.QueryRow(
		`SELECT t.id, t.title, COALESCE(t.motivation,''), COALESCE(t.tags,'[]'), COALESCE(t.updated_at,'')
		 FROM threads t JOIN settings s ON s.value = CAST(t.id AS TEXT)
		 WHERE s.key='golden_thread_id'`,
	).Scan(&id, &title, &motivation, &tags, &updatedAt)
	if err != nil {
		return 0, nil, nil
	}

	kwCache.Lock()
	if kwCache.threadID == id && kwCache.updatedAt == updatedAt {
		kws, bgs := kwCache.keywords, kwCache.bigrams
		kwCache.Unlock()
		return id, kws, bgs
	}
	kwCache.Unlock()

	raw := extractKeywords(title + " " + motivation + " " + tags)
	filtered := k.filterByDocFreq(tx, raw)

	// Keep only bigrams where both words survived DF filtering.
	survived := make(map[string]bool, len(filtered))
	for _, kw := range filtered {
		survived[kw] = true
	}
	var bigrams [][2]string
	for _, bg := range extractBigrams(title + " " + motivation + " " + tags) {
		if survived[bg[0]] && survived[bg[1]] {
			bigrams = append(bigrams, bg)
		}
	}

	kwCache.Lock()
	kwCache.threadID, kwCache.updatedAt = id, updatedAt
	kwCache.keywords, kwCache.bigrams = filtered, bigrams
	kwCache.Unlock()
	return id, filtered, bigrams
}

// filterByDocFreq drops keywords that match more than maxDocFreq of all entries.
// Rare keywords (klubhaus, doorbell) are discriminative; common ones (board,
// current) just link noise.
func (k *KnoxDB) filterByDocFreq(tx *sql.Tx, keywords []string) []string {
	var total int
	if err := tx.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 {
		return keywords
	}
	limit := int(float64(total) * maxDocFreq)
	if limit < 5 {
		limit = 5
	}
	var out []string
	for _, kw := range keywords {
		var df int
		like := "%" + escapeLike(kw) + "%"
		err := tx.QueryRow(
			`SELECT COUNT(*) FROM entries
			 WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`,
			like, like, like,
		).Scan(&df)
		if err != nil || df <= limit {
			out = append(out, kw)
		}
	}
	return out
}

var wordRE = regexp.MustCompile(`[^a-z0-9]+`)

// wordBoundaryRE extracts maximal alphanumeric runs. No \b anchors — Go treats
// underscore as a word char, so \b would swallow snake_case tokens entirely
// ("ALERT_klubhaus_topic" would yield nothing).
var wordBoundaryRE = regexp.MustCompile(`[a-z0-9]+`)

var stopWords = map[string]bool{
	"with": true, "this": true, "that": true, "from": true,
	"they": true, "have": true, "were": true, "what": true,
	"when": true, "where": true, "which": true, "their": true,
	"them": true, "been": true, "also": true, "than": true,
	"into": true, "over": true, "such": true, "each": true,
	"about": true, "most": true, "some": true, "more": true,
	"other": true, "then": true, "will": true, "just": true,
	"like": true, "only": true, "very": true, "even": true,
	"much": true, "still": true, "well": true, "here": true,
	"there": true, "does": true, "done": true, "system": true,
	"page": true, "site": true, "user": true, "data": true,
	"file": true, "type": true, "home": true, "mode": true,
	"show": true, "name": true, "back": true, "open": true,
}

func ExtractKeywords(s string) []string { return extractKeywords(s) }
func extractKeywords(s string) []string {
	s = strings.ToLower(s)
	words := wordRE.Split(s, -1)
	seen := make(map[string]bool)
	var keywords []string
	for _, w := range words {
		if len(w) > 3 && !seen[w] && !stopWords[w] {
			seen[w] = true
			keywords = append(keywords, w)
		}
	}
	return keywords
}

// extractBigrams returns ordered consecutive keyword pairs from the text,
// preserving phrase structure ("multi board" from "multi-board").
func extractBigrams(s string) [][2]string {
	s = strings.ToLower(s)
	words := wordRE.Split(s, -1)
	var kept []string
	for _, w := range words {
		if len(w) > 3 && !stopWords[w] {
			kept = append(kept, w)
		}
	}
	seen := make(map[[2]string]bool)
	var out [][2]string
	for i := 0; i+1 < len(kept); i++ {
		pair := [2]string{kept[i], kept[i+1]}
		if !seen[pair] {
			seen[pair] = true
			out = append(out, pair)
		}
	}
	return out
}

// isRelevant scores phrase and keyword overlap:
//   - a bigram phrase match (in title or body) scores 3 — "multi board" is
//     precise, "board" alone is not
//   - a keyword in the title scores 2
//   - a keyword in summary/project scores 1
//
// Threshold is 3, so a single weak keyword never links by itself.
func isRelevant(title, summary, project string, keywords []string, bigrams [][2]string) bool {
	titleTokens := wordBoundaryRE.FindAllString(strings.ToLower(title), -1)
	bodyTokens := wordBoundaryRE.FindAllString(strings.ToLower(summary+" "+project), -1)

	titleSet := make(map[string]bool, len(titleTokens))
	for _, t := range titleTokens {
		titleSet[t] = true
	}
	bodySet := make(map[string]bool, len(bodyTokens))
	for _, t := range bodyTokens {
		bodySet[t] = true
	}

	score := 0
	for _, kw := range keywords {
		if titleSet[kw] {
			score += 2
		} else if bodySet[kw] {
			score += 1
		}
	}
	for _, bg := range bigrams {
		if containsBigram(titleTokens, bg) || containsBigram(bodyTokens, bg) {
			score += 3
		}
	}
	return score >= 3
}

func containsBigram(tokens []string, bg [2]string) bool {
	for i := 0; i+1 < len(tokens); i++ {
		if tokens[i] == bg[0] && tokens[i+1] == bg[1] {
			return true
		}
	}
	return false
}

// ─── Thread (Motivation/Intent) Operations ─────────────────────

type Thread struct {
	ID         int64
	Title      string
	Motivation string
	Status     string
	Priority   string
	CreatedAt  string
	UpdatedAt  string
	ResolvedAt string
	Tags       string
	Provenance string
	EntryCount int
}

func ThreadFP(id int64) string { return fmt.Sprintf("thread:%d", id) }

// CreateThread inserts a thread. clusterKey, when non-empty, provides
// idempotency: if a thread with that cluster key already exists no new row is
// created. The existing id is returned with created=false.
func (k *KnoxDB) CreateThread(title, motivation, priority, tags, provenance string) (int64, error) {
	id, _, err := k.CreateThreadCluster(title, motivation, priority, tags, provenance, "")
	return id, err
}

func (k *KnoxDB) CreateThreadCluster(title, motivation, priority, tags, provenance, clusterKey string) (int64, bool, error) {
	if clusterKey != "" {
		if existing := k.ThreadByClusterKey(clusterKey); existing != 0 {
			return existing, false, nil
		}
	}
	res, err := k.db.Exec(
		`INSERT INTO threads (title, motivation, priority, tags, provenance, cluster_key) VALUES (?, ?, ?, ?, ?, ?)`,
		title, motivation, priority, tags, provenance, clusterKey,
	)
	if err != nil {
		return 0, false, err
	}
	id, _ := res.LastInsertId()
	return id, true, nil
}

// ThreadByClusterKey returns the id of the thread auto-created for a given
// cluster, or 0 when none exists.
func (k *KnoxDB) ThreadByClusterKey(clusterKey string) int64 {
	if clusterKey == "" {
		return 0
	}
	var id int64
	err := k.db.QueryRow(`SELECT id FROM threads WHERE cluster_key=?`, clusterKey).Scan(&id)
	if err != nil {
		return 0
	}
	return id
}

func (k *KnoxDB) ListThreads(status string) ([]Thread, error) {
	query := `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority,
	                 t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'),
	                 COALESCE(t.provenance,'{}'),
	                 (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id)
	          FROM threads t`
	var args []any
	if status != "" {
		query += " WHERE t.status=?"
		args = append(args, status)
	}
	query += " ORDER BY t.updated_at DESC"

	rows, err := k.db.Query(query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var threads []Thread
	for rows.Next() {
		var t Thread
		if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority,
			&t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil {
			return nil, err
		}
		threads = append(threads, t)
	}
	return threads, nil
}

func (k *KnoxDB) GetThread(id int64) (*Thread, error) {
	row := k.db.QueryRow(
		`SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority,
		        t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'),
		        COALESCE(t.provenance,'{}'),
		        (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id)
		 FROM threads t WHERE t.id=?`, id,
	)
	t := &Thread{}
	err := row.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority,
		&t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount)
	if err == sql.ErrNoRows {
		return nil, nil
	}
	return t, err
}

func (k *KnoxDB) CloseThread(id int64) error {
	_, err := k.db.Exec(
		`UPDATE threads SET status='resolved', resolved_at=datetime('now'), updated_at=datetime('now') WHERE id=?`,
		id,
	)
	return err
}

// UpdateThread rewrites the editorial fields of a thread (used by the LLM
// draft/polish pass to replace auto-generated titles/motivations). Empty
// strings leave the corresponding field unchanged; omitting them all is a no-op.
func (k *KnoxDB) UpdateThread(id int64, title, motivation, priority, tags string) (bool, error) {
	t, err := k.GetThread(id)
	if err != nil || t == nil {
		return false, fmt.Errorf("thread #%d not found", id)
	}
	if title == "" {
		title = t.Title
	}
	if motivation == "" {
		motivation = t.Motivation
	}
	if priority == "" {
		priority = t.Priority
	}
	if tags == "" {
		tags = t.Tags
	}
	changed := title != t.Title || motivation != t.Motivation || priority != t.Priority || tags != t.Tags
	if !changed {
		return false, nil
	}
	_, err = k.db.Exec(
		`UPDATE threads SET title=?, motivation=?, priority=?, tags=?, updated_at=datetime('now') WHERE id=?`,
		title, motivation, priority, tags, id,
	)
	return true, err
}

func (k *KnoxDB) LinkObservationToThread(threadID, observationID int64, relevance string) error {
	_, err := k.db.Exec(
		`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, ?)`,
		threadID, observationID, relevance,
	)
	if err != nil {
		return err
	}
	_, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID)
	return err
}

// AutoLinkThreadObservations resolves each entry fingerprint to its most recent
// observation and links it to the thread ('auto_cluster' relevance). Idempotent:
// INSERT OR IGNORE means re-linking an already-linked fingerprint is a no-op.
// Returns the number of newly linked observations.
func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []string) (int, error) {
	tx, err := k.db.Begin()
	if err != nil {
		return 0, err
	}
	defer tx.Rollback()

	linked := 0
	for _, fp := range fingerprints {
		if fp == "" {
			continue
		}
		var obsID int64
		if err := tx.QueryRow(
			`SELECT id FROM observations WHERE fingerprint=? ORDER BY hcl DESC, id DESC LIMIT 1`, fp,
		).Scan(&obsID); err != nil {
			continue
		}
		res, err := tx.Exec(
			`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto_cluster')`,
			threadID, obsID,
		)
		if err != nil {
			continue
		}
		if n, _ := res.RowsAffected(); n > 0 {
			linked++
		}
	}

	if linked > 0 {
		if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID); err != nil {
			return linked, err
		}
	}
	return linked, tx.Commit()
}

// ActiveThreadByKeyword returns the most recently updated active thread whose
// title/motivation/tags contains the given keyword, or 0 if none exists. Used
// to fold new clusters into an existing thread instead of spawning duplicates.
func (k *KnoxDB) ActiveThreadByKeyword(keyword string) (int64, error) {
	if keyword == "" {
		return 0, nil
	}
	like := "%" + escapeLike(keyword) + "%"
	rows, err := k.db.Query(
		`SELECT id FROM threads
		 WHERE status='active' AND (title LIKE ? ESCAPE '\' OR motivation LIKE ? ESCAPE '\' OR tags LIKE ? ESCAPE '\')
		 ORDER BY updated_at DESC LIMIT 1`,
		like, like, like,
	)
	if err != nil {
		return 0, err
	}
	defer rows.Close()
	if rows.Next() {
		var id int64
		if err := rows.Scan(&id); err != nil {
			return 0, err
		}
		return id, nil
	}
	return 0, rows.Err()
}

// ThreadObservations returns linked observations, deduplicated by fingerprint
// (re-observed entries appear once, with the latest observation's data).
func (k *KnoxDB) ThreadObservations(threadID int64) ([]Observation, error) {
	rows, err := k.db.Query(
		`SELECT o.id, o.fingerprint, o.source_id, COALESCE(o.source_path,''), COALESCE(o.project,''),
		        COALESCE(o.content_type,''), COALESCE(o.title,''), COALESCE(o.summary,''),
		        o.collected_at, COALESCE(o.created_at,''), COALESCE(o.line_start,0), COALESCE(o.line_end,0),
		        o.confidence, o.ingester_version, COALESCE(o.trigger,'')
		 FROM observations o
		 JOIN thread_observations to2 ON to2.observation_id = o.id
		 WHERE to2.thread_id=?
		   AND o.id IN (
		     SELECT MAX(to3.observation_id) FROM thread_observations to3
		     JOIN observations o2 ON o2.id = to3.observation_id
		     WHERE to3.thread_id=?
		     GROUP BY o2.fingerprint
		   )
		 ORDER BY COALESCE(NULLIF(o.created_at,''), o.collected_at) DESC`, threadID, threadID,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var obs []Observation
	for rows.Next() {
		var o Observation
		if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType,
			&o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd,
			&o.Confidence, &o.IngesterVersion, &o.Trigger); err != nil {
			return nil, err
		}
		obs = append(obs, o)
	}
	return obs, nil
}

func (k *KnoxDB) AddThreadNote(threadID int64, note string) (int64, error) {
	res, err := k.db.Exec(
		`INSERT INTO thread_notes (thread_id, note) VALUES (?, ?)`,
		threadID, note,
	)
	if err != nil {
		return 0, err
	}
	_, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID)
	return res.LastInsertId()
}

func (k *KnoxDB) ThreadNotes(threadID int64) ([]struct {
	ID        int64
	Note      string
	CreatedAt string
}, error) {
	rows, err := k.db.Query(
		`SELECT id, note, created_at FROM thread_notes WHERE thread_id=? ORDER BY created_at`, threadID,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var notes []struct {
		ID        int64
		Note      string
		CreatedAt string
	}
	for rows.Next() {
		var n struct {
			ID        int64
			Note      string
			CreatedAt string
		}
		if err := rows.Scan(&n.ID, &n.Note, &n.CreatedAt); err != nil {
			return nil, err
		}
		notes = append(notes, n)
	}
	return notes, nil
}

func (k *KnoxDB) SearchThreadsByMotivation(query string) ([]Thread, error) {
	like := "%" + escapeLike(query) + "%"
	rows, err := k.db.Query(
		`SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority,
		        t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'),
		        COALESCE(t.provenance,'{}'),
		        (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id)
		 FROM threads t
		 WHERE t.title LIKE ? ESCAPE '\' OR t.motivation LIKE ? ESCAPE '\' OR t.tags LIKE ? ESCAPE '\'
		 ORDER BY t.updated_at DESC LIMIT 20`,
		like, like, like,
	)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var threads []Thread
	for rows.Next() {
		var t Thread
		if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority,
			&t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil {
			return nil, err
		}
		threads = append(threads, t)
	}
	return threads, nil
}

// LinkEntryToThread links an entry to a thread via provenance_links.
func (k *KnoxDB) LinkEntryToThread(threadID int64, entryFP, relation string) error {
	if entryFP == "" {
		return fmt.Errorf("entry fingerprint must not be empty")
	}
	tfp := ThreadFP(threadID)
	_, err := k.db.Exec(
		`INSERT OR IGNORE INTO provenance_links (descendant_fp, ancestor_fp, hop_distance, relation) VALUES (?, ?, 1, ?)`,
		entryFP, tfp, relation,
	)
	return err
}

// ThreadRelevanceProfile returns the DF-filtered keywords and bigrams for any thread.
func (k *KnoxDB) ThreadRelevanceProfile(threadID int64) ([]string, [][2]string, error) {
	var title, motivation, tags string
	err := k.db.QueryRow(
		`SELECT title, COALESCE(motivation,''), COALESCE(tags,'[]') FROM threads WHERE id=?`, threadID,
	).Scan(&title, &motivation, &tags)
	if err != nil {
		return nil, nil, err
	}
	raw := extractKeywords(title + " " + motivation + " " + tags)

	var total int
	if err := k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 {
		return raw, nil, nil
	}
	limit := int(float64(total) * maxDocFreq)
	if limit < 5 {
		limit = 5
	}
	survived := make(map[string]bool)
	var keywords []string
	for _, kw := range raw {
		var df int
		like := "%" + escapeLike(kw) + "%"
		if err := k.db.QueryRow(
			`SELECT COUNT(*) FROM entries
			 WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`,
			like, like, like,
		).Scan(&df); err != nil || df <= limit {
			keywords = append(keywords, kw)
			survived[kw] = true
		}
	}
	var bigrams [][2]string
	for _, bg := range extractBigrams(title + " " + motivation + " " + tags) {
		if survived[bg[0]] && survived[bg[1]] {
			bigrams = append(bigrams, bg)
		}
	}
	return keywords, bigrams, nil
}

// PruneThread removes machine-linked observations ('auto' and legacy 'regroom')
// that fail the (stricter) relevance scoring, using the thread's DF-filtered
// keyword profile. Temporal links and manual links (descriptive relevance
// strings) are evidence of a different kind and are left intact.
func (k *KnoxDB) PruneThread(threadID int64) (int, error) {
	keywords, bigrams, err := k.ThreadRelevanceProfile(threadID)
	if err != nil {
		return 0, err
	}
	if len(keywords) == 0 {
		return 0, nil
	}
	rows, err := k.db.Query(
		`SELECT to2.observation_id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'')
		 FROM thread_observations to2
		 JOIN observations o ON o.id = to2.observation_id
		 WHERE to2.thread_id = ? AND to2.relevance IN ('auto', 'regroom')`,
		threadID,
	)
	if err != nil {
		return 0, err
	}

	var staleIDs []int64
	for rows.Next() {
		var id int64
		var title, summary, project string
		if err := rows.Scan(&id, &title, &summary, &project); err != nil {
			rows.Close()
			return 0, err
		}
		if !isRelevant(title, summary, project, keywords, bigrams) {
			staleIDs = append(staleIDs, id)
		}
	}
	rows.Close()

	if len(staleIDs) == 0 {
		return 0, nil
	}
	placeholders := strings.TrimSuffix(strings.Repeat("?,", len(staleIDs)), ",")
	args := make([]any, 0, len(staleIDs)+1)
	args = append(args, threadID)
	for _, id := range staleIDs {
		args = append(args, id)
	}
	res, err := k.db.Exec(
		fmt.Sprintf(`DELETE FROM thread_observations WHERE thread_id=? AND observation_id IN (%s)`, placeholders),
		args...,
	)
	if err != nil {
		return 0, err
	}
	n, _ := res.RowsAffected()
	return int(n), nil
}

// RescanThread links existing observations that pass the thread's relevance
// profile but were never linked — the inverse of PruneThread. One link row per
// fingerprint (latest observation), relevance 'auto'.
func (k *KnoxDB) RescanThread(threadID int64) (int, error) {
	keywords, bigrams, err := k.ThreadRelevanceProfile(threadID)
	if err != nil {
		return 0, err
	}
	if len(keywords) == 0 {
		return 0, nil
	}
	rows, err := k.db.Query(
		`SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'')
		 FROM observations o
		 WHERE o.id IN (SELECT MAX(id) FROM observations GROUP BY fingerprint)
		   AND o.fingerprint NOT IN (
		     SELECT o2.fingerprint FROM thread_observations t
		     JOIN observations o2 ON o2.id = t.observation_id WHERE t.thread_id = ?
		   )`,
		threadID,
	)
	if err != nil {
		return 0, err
	}

	var linkIDs []int64
	for rows.Next() {
		var id int64
		var title, summary, project string
		if err := rows.Scan(&id, &title, &summary, &project); err != nil {
			rows.Close()
			return 0, err
		}
		if isRelevant(title, summary, project, keywords, bigrams) {
			linkIDs = append(linkIDs, id)
		}
	}
	rows.Close()

	linked := 0
	for _, id := range linkIDs {
		res, err := k.db.Exec(
			`INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`,
			threadID, id,
		)
		if err == nil {
			if n, _ := res.RowsAffected(); n > 0 {
				linked++
			}
		}
	}
	return linked, nil
}

// ThreadProvenance returns the provenance graph for a thread.
func (k *KnoxDB) ThreadProvenance(threadID int64) ([]struct {
	Fingerprint string
	Relation    string
	HopDistance int
}, error) {
	return k.ProvenanceChain(ThreadFP(threadID), 3)
}