knox/internal/watch/threader.go

package watch

import (
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/david/knox/internal/db"
	"github.com/david/knox/internal/index"
)

// AutoThreader promotes high-intent topic clusters into knowledge threads.
//
// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the
// "intent bar" when it shows sustained, recently-observed activity from a
// non-passive work source (git/gitea/session/obsidian) plus enough distinct
// sources overall. Motivation and title are synthesized from the cluster's
// strongest members — they can be refined later, but the grouping and linkage
// are deterministic and cheap enough for the watch daemon's periodic tick.
type AutoThreader struct {
	DB         *db.KnoxDB
	MinSources int           // distinct sources required (incl. one work source)
	MinEntries int           // minimum cluster size
	Recency    time.Duration // observations must be seen within this window
	DryRun     bool
}

// workSources are signal sources that imply active intent (vs passive browsing
// or an always-present skills catalog).
var workSources = map[string]bool{
	"git":              true,
	"gitea":            true,
	"opencode-session": true,
	"obsidian":         true,
	"filesystem":       true,
}

// NewAutoThreader reads the tuning knobs from the environment. All have
// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off.
func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader {
	t := &AutoThreader{
		DB:         kdb,
		MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2),
		MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3),
		Recency:    time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour,
	}
	return t
}

func envInt(key string, def int) int {
	if v, err := strconv.Atoi(os.Getenv(key)); err == nil {
		return v
	}
	return def
}

// AutoThread clusters recent entries and creates/extends threads for clusters
// crossing the intent bar. Returns the number of threads created and the total
// observations linked.
func (t *AutoThreader) AutoThread() (created, linked int, err error) {
	entries, err := t.DB.RecentEntries(2000)
	if err != nil {
		return 0, 0, err
	}
	if len(entries) == 0 {
		return 0, 0, nil
	}

	tfidf := index.BuildTFIDF(entries)
	clusters := tfidf.Cluster(2, 30)
	now := time.Now().UTC()

	for _, c := range clusters {
		fps := fingerprints(c.Entries)
		if len(fps) == 0 || !t.crossesBar(c, fps, now) {
			continue
		}

		// Fold into the exact thread this cluster keyed to (idempotent across
		// nodes/reconciles); otherwise fall back to the keyword heuristic which
		// also matches human-edited threads that keep their auto cluster key.
		existing := t.DB.ThreadByClusterKey(clusterKey(c))
		if existing == 0 {
			existing = t.findExisting(c)
		}
		if existing > 0 {
			if t.DryRun {
				fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name)
				continue
			}
			n, err := t.DB.AutoLinkThreadObservations(existing, fps)
			if err != nil {
				log.Printf("[knox] threader link err: %v", err)
				continue
			}
			if n > 0 {
				log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n)
			}
			linked += n
			continue
		}

		title := t.title(c)
		motivation := t.motivation(c, fps)
		tags := t.tags(c)
		priority := t.priority(c)
		prov := map[string]any{
			"trigger": "auto_thread",
			"cluster": c.Name,
			"score":   c.Score,
			"sources": sourceList(c.Entries),
		}
		provJSON, _ := json.Marshal(prov)

		if t.DryRun {
			fmt.Printf("[threader] (dry-run) would create thread %q  [%s]\n", title, priority)
			fmt.Printf("            motivation: %s\n", motivation)
			fmt.Printf("            tags: %s\n", tags)
			fmt.Printf("            %d obs | sources: %v\n", len(fps), prov["sources"])
			continue
		}

		id, isNew, err := t.DB.CreateThreadCluster(title, motivation, priority, tags, string(provJSON), clusterKey(c))
		if err != nil {
			log.Printf("[knox] threader create err: %v", err)
			continue
		}
		if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil {
			linked += n
		}
		if isNew {
			created++
		}
		log.Printf("[knox] auto-thread %s #%d: %s (%d obs, %s)", statusWord(isNew), id, title, len(fps), priority)
	}
	return created, linked, nil
}

func statusWord(isNew bool) string {
	if isNew {
		return "created"
	}
	return "extended"
}

// clusterKey returns a deterministic content hash keying a thread to the topic
// cluster that generated it. Sorted keywords make it machine-independent, so
// two nodes auto-creating the same cluster converge on the same key (idempotent
// thread creation via CreateThreadCluster).
func clusterKey(c index.TopicCluster) string {
	kws := make([]string, len(c.Keywords))
	copy(kws, c.Keywords)
	sort.Strings(kws)
	joined := "thread:cluster:" + strings.Join(kws, "\x00")
	sum := sha256.Sum256([]byte(joined))
	return fmt.Sprintf("cluster:%x", sum[:16])
}

// crossesBar decides whether a cluster reflects real intent.
func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool {
	if len(c.Entries) < t.MinEntries {
		return false
	}

	bySource := make(map[string]bool)
	hasWork := false
	recent := false
	for _, e := range c.Entries {
		if e.SourceID != "" {
			bySource[e.SourceID] = true
			if workSources[e.SourceID] {
				hasWork = true
			}
		}
		if e.LastSeen != "" {
			if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency {
				recent = true
			}
		}
	}

	if len(bySource) < t.MinSources {
		return false
	}
	if !hasWork {
		return false
	}
	return recent
}

// findExisting looks for an already-active thread that plausibly covers this
// cluster, by matching a discriminative cluster keyword against thread titles.
func (t *AutoThreader) findExisting(c index.TopicCluster) int64 {
	for _, kw := range c.Keywords {
		id, err := t.DB.ActiveThreadByKeyword(kw)
		if err != nil {
			return 0
		}
		if id == 0 {
			continue
		}
		th, err := t.DB.GetThread(id)
		if err != nil || th == nil {
			continue
		}
		titleTokens := toSet(strings.ToLower(th.Title))
		for _, kw2 := range c.Keywords {
			if titleTokens[kw2] {
				return id
			}
		}
	}
	return 0
}

func (t *AutoThreader) title(c index.TopicCluster) string {
	return c.Name
}

func (t *AutoThreader) tags(c index.TopicCluster) string {
	var keep []string
	for _, kw := range c.Keywords {
		if len(keep) >= 5 {
			break
		}
		keep = append(keep, kw)
	}
	return strings.Join(keep, " ")
}

func (t *AutoThreader) priority(c index.TopicCluster) string {
	srcs := len(sourceList(c.Entries))
	if srcs >= 3 {
		return "high"
	}
	return "medium"
}

// motivation synthesizes "why" from the cluster's strongest members: project
// context, source spread, and a couple of representative titles/summaries.
func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string {
	sources := sourceList(c.Entries)

	proj := dominantProject(c.Entries)

	var b strings.Builder
	fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", "))
	if proj != "" {
		fmt.Fprintf(&b, " Dominant project: %s.", proj)
	}
	fmt.Fprintf(&b, " Keywords: %s.", c.Name)
	fmt.Fprintf(&b, " Representative observations:")

	shown := 0
	seen := make(map[string]bool)
	for _, e := range c.Entries {
		if shown >= 3 {
			break
		}
		txt := strings.TrimSpace(e.Title)
		if txt == "" {
			txt = strings.TrimSpace(e.Summary)
		}
		if txt == "" || seen[txt] {
			continue
		}
		seen[txt] = true
		fmt.Fprintf(&b, " • %s", truncate(txt, 90))
		shown++
	}
	return b.String()
}

// --- small helpers ---------------------------------------------------------

func fingerprints(entries []db.Entry) []string {
	seen := make(map[string]bool)
	var out []string
	for _, e := range entries {
		if e.Fingerprint == "" || seen[e.Fingerprint] {
			continue
		}
		seen[e.Fingerprint] = true
		out = append(out, e.Fingerprint)
	}
	return out
}

func sourceList(entries []db.Entry) []string {
	seen := make(map[string]bool)
	var out []string
	for _, e := range entries {
		if e.SourceID == "" || seen[e.SourceID] {
			continue
		}
		seen[e.SourceID] = true
		out = append(out, e.SourceID)
	}
	return out
}

func dominantProject(entries []db.Entry) string {
	counts := make(map[string]int)
	for _, e := range entries {
		if e.Project != "" {
			counts[e.Project]++
		}
	}
	best, n := "", 0
	for p, c := range counts {
		if c > n {
			best, n = p, c
		}
	}
	if n*2 >= len(entries) {
		return best
	}
	return ""
}

func toSet(s string) map[string]bool {
	set := make(map[string]bool)
	for _, w := range strings.Fields(s) {
		set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true
	}
	return set
}

func truncate(s string, n int) string {
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return string(r[:n]) + "…"
}