package watch import ( "encoding/json" "fmt" "log" "os" "strconv" "strings" "time" "github.com/david/knox/internal/db" "github.com/david/knox/internal/index" ) // AutoThreader promotes high-intent topic clusters into knowledge threads. // // Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the // "intent bar" when it shows sustained, recently-observed activity from a // non-passive work source (git/gitea/session/obsidian) plus enough distinct // sources overall. Motivation and title are synthesized from the cluster's // strongest members — they can be refined later, but the grouping and linkage // are deterministic and cheap enough for the watch daemon's periodic tick. type AutoThreader struct { DB *db.KnoxDB MinSources int // distinct sources required (incl. one work source) MinEntries int // minimum cluster size Recency time.Duration // observations must be seen within this window DryRun bool } // workSources are signal sources that imply active intent (vs passive browsing // or an always-present skills catalog). var workSources = map[string]bool{ "git": true, "gitea": true, "opencode-session": true, "obsidian": true, "filesystem": true, } // NewAutoThreader reads the tuning knobs from the environment. All have // defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off. func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader { t := &AutoThreader{ DB: kdb, MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2), MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3), Recency: time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour, } return t } func envInt(key string, def int) int { if v, err := strconv.Atoi(os.Getenv(key)); err == nil { return v } return def } // AutoThread clusters recent entries and creates/extends threads for clusters // crossing the intent bar. Returns the number of threads created and the total // observations linked. func (t *AutoThreader) AutoThread() (created, linked int, err error) { entries, err := t.DB.RecentEntries(2000) if err != nil { return 0, 0, err } if len(entries) == 0 { return 0, 0, nil } tfidf := index.BuildTFIDF(entries) clusters := tfidf.Cluster(2, 30) now := time.Now().UTC() for _, c := range clusters { fps := fingerprints(c.Entries) if len(fps) == 0 || !t.crossesBar(c, fps, now) { continue } // Fold into an existing active thread if one already covers these terms. if existing := t.findExisting(c); existing > 0 { if t.DryRun { fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name) continue } n, err := t.DB.AutoLinkThreadObservations(existing, fps) if err != nil { log.Printf("[knox] threader link err: %v", err) continue } if n > 0 { log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n) } linked += n continue } title := t.title(c) motivation := t.motivation(c, fps) tags := t.tags(c) priority := t.priority(c) prov := map[string]any{ "trigger": "auto_thread", "cluster": c.Name, "score": c.Score, "sources": sourceList(c.Entries), } provJSON, _ := json.Marshal(prov) if t.DryRun { fmt.Printf("[threader] (dry-run) would create thread %q [%s]\n", title, priority) fmt.Printf(" motivation: %s\n", motivation) fmt.Printf(" tags: %s\n", tags) fmt.Printf(" %d obs | sources: %v\n", len(fps), prov["sources"]) continue } id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON)) if err != nil { log.Printf("[knox] threader create err: %v", err) continue } if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil { linked += n } created++ log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority) } return created, linked, nil } // crossesBar decides whether a cluster reflects real intent. func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool { if len(c.Entries) < t.MinEntries { return false } bySource := make(map[string]bool) hasWork := false recent := false for _, e := range c.Entries { if e.SourceID != "" { bySource[e.SourceID] = true if workSources[e.SourceID] { hasWork = true } } if e.LastSeen != "" { if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency { recent = true } } } if len(bySource) < t.MinSources { return false } if !hasWork { return false } return recent } // findExisting looks for an already-active thread that plausibly covers this // cluster, by matching a discriminative cluster keyword against thread titles. func (t *AutoThreader) findExisting(c index.TopicCluster) int64 { for _, kw := range c.Keywords { id, err := t.DB.ActiveThreadByKeyword(kw) if err != nil { return 0 } if id == 0 { continue } th, err := t.DB.GetThread(id) if err != nil || th == nil { continue } titleTokens := toSet(strings.ToLower(th.Title)) for _, kw2 := range c.Keywords { if titleTokens[kw2] { return id } } } return 0 } func (t *AutoThreader) title(c index.TopicCluster) string { return c.Name } func (t *AutoThreader) tags(c index.TopicCluster) string { var keep []string for _, kw := range c.Keywords { if len(keep) >= 5 { break } keep = append(keep, kw) } return strings.Join(keep, " ") } func (t *AutoThreader) priority(c index.TopicCluster) string { srcs := len(sourceList(c.Entries)) if srcs >= 3 { return "high" } return "medium" } // motivation synthesizes "why" from the cluster's strongest members: project // context, source spread, and a couple of representative titles/summaries. func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string { sources := sourceList(c.Entries) proj := dominantProject(c.Entries) var b strings.Builder fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", ")) if proj != "" { fmt.Fprintf(&b, " Dominant project: %s.", proj) } fmt.Fprintf(&b, " Keywords: %s.", c.Name) fmt.Fprintf(&b, " Representative observations:") shown := 0 seen := make(map[string]bool) for _, e := range c.Entries { if shown >= 3 { break } txt := strings.TrimSpace(e.Title) if txt == "" { txt = strings.TrimSpace(e.Summary) } if txt == "" || seen[txt] { continue } seen[txt] = true fmt.Fprintf(&b, " • %s", truncate(txt, 90)) shown++ } return b.String() } // --- small helpers --------------------------------------------------------- func fingerprints(entries []db.Entry) []string { seen := make(map[string]bool) var out []string for _, e := range entries { if e.Fingerprint == "" || seen[e.Fingerprint] { continue } seen[e.Fingerprint] = true out = append(out, e.Fingerprint) } return out } func sourceList(entries []db.Entry) []string { seen := make(map[string]bool) var out []string for _, e := range entries { if e.SourceID == "" || seen[e.SourceID] { continue } seen[e.SourceID] = true out = append(out, e.SourceID) } return out } func dominantProject(entries []db.Entry) string { counts := make(map[string]int) for _, e := range entries { if e.Project != "" { counts[e.Project]++ } } best, n := "", 0 for p, c := range counts { if c > n { best, n = p, c } } if n*2 >= len(entries) { return best } return "" } func toSet(s string) map[string]bool { set := make(map[string]bool) for _, w := range strings.Fields(s) { set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true } return set } func truncate(s string, n int) string { r := []rune(s) if len(r) <= n { return s } return string(r[:n]) + "…" }