Initial commit: knox knowledge index
This commit is contained in:
@@ -0,0 +1,311 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/index"
|
||||
)
|
||||
|
||||
// AutoThreader promotes high-intent topic clusters into knowledge threads.
|
||||
//
|
||||
// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the
|
||||
// "intent bar" when it shows sustained, recently-observed activity from a
|
||||
// non-passive work source (git/gitea/session/obsidian) plus enough distinct
|
||||
// sources overall. Motivation and title are synthesized from the cluster's
|
||||
// strongest members — they can be refined later, but the grouping and linkage
|
||||
// are deterministic and cheap enough for the watch daemon's periodic tick.
|
||||
type AutoThreader struct {
|
||||
DB *db.KnoxDB
|
||||
MinSources int // distinct sources required (incl. one work source)
|
||||
MinEntries int // minimum cluster size
|
||||
Recency time.Duration // observations must be seen within this window
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// workSources are signal sources that imply active intent (vs passive browsing
|
||||
// or an always-present skills catalog).
|
||||
var workSources = map[string]bool{
|
||||
"git": true,
|
||||
"gitea": true,
|
||||
"opencode-session": true,
|
||||
"obsidian": true,
|
||||
"filesystem": true,
|
||||
}
|
||||
|
||||
// NewAutoThreader reads the tuning knobs from the environment. All have
|
||||
// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off.
|
||||
func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader {
|
||||
t := &AutoThreader{
|
||||
DB: kdb,
|
||||
MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2),
|
||||
MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3),
|
||||
Recency: time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour,
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
if v, err := strconv.Atoi(os.Getenv(key)); err == nil {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// AutoThread clusters recent entries and creates/extends threads for clusters
|
||||
// crossing the intent bar. Returns the number of threads created and the total
|
||||
// observations linked.
|
||||
func (t *AutoThreader) AutoThread() (created, linked int, err error) {
|
||||
entries, err := t.DB.RecentEntries(2000)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, 30)
|
||||
now := time.Now().UTC()
|
||||
|
||||
for _, c := range clusters {
|
||||
fps := fingerprints(c.Entries)
|
||||
if len(fps) == 0 || !t.crossesBar(c, fps, now) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Fold into an existing active thread if one already covers these terms.
|
||||
if existing := t.findExisting(c); existing > 0 {
|
||||
if t.DryRun {
|
||||
fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name)
|
||||
continue
|
||||
}
|
||||
n, err := t.DB.AutoLinkThreadObservations(existing, fps)
|
||||
if err != nil {
|
||||
log.Printf("[knox] threader link err: %v", err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n)
|
||||
}
|
||||
linked += n
|
||||
continue
|
||||
}
|
||||
|
||||
title := t.title(c)
|
||||
motivation := t.motivation(c, fps)
|
||||
tags := t.tags(c)
|
||||
priority := t.priority(c)
|
||||
prov := map[string]any{
|
||||
"trigger": "auto_thread",
|
||||
"cluster": c.Name,
|
||||
"score": c.Score,
|
||||
"sources": sourceList(c.Entries),
|
||||
}
|
||||
provJSON, _ := json.Marshal(prov)
|
||||
|
||||
if t.DryRun {
|
||||
fmt.Printf("[threader] (dry-run) would create thread %q [%s]\n", title, priority)
|
||||
fmt.Printf(" motivation: %s\n", motivation)
|
||||
fmt.Printf(" tags: %s\n", tags)
|
||||
fmt.Printf(" %d obs | sources: %v\n", len(fps), prov["sources"])
|
||||
continue
|
||||
}
|
||||
|
||||
id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON))
|
||||
if err != nil {
|
||||
log.Printf("[knox] threader create err: %v", err)
|
||||
continue
|
||||
}
|
||||
if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil {
|
||||
linked += n
|
||||
}
|
||||
created++
|
||||
log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority)
|
||||
}
|
||||
return created, linked, nil
|
||||
}
|
||||
|
||||
// crossesBar decides whether a cluster reflects real intent.
|
||||
func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool {
|
||||
if len(c.Entries) < t.MinEntries {
|
||||
return false
|
||||
}
|
||||
|
||||
bySource := make(map[string]bool)
|
||||
hasWork := false
|
||||
recent := false
|
||||
for _, e := range c.Entries {
|
||||
if e.SourceID != "" {
|
||||
bySource[e.SourceID] = true
|
||||
if workSources[e.SourceID] {
|
||||
hasWork = true
|
||||
}
|
||||
}
|
||||
if e.LastSeen != "" {
|
||||
if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency {
|
||||
recent = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(bySource) < t.MinSources {
|
||||
return false
|
||||
}
|
||||
if !hasWork {
|
||||
return false
|
||||
}
|
||||
return recent
|
||||
}
|
||||
|
||||
// findExisting looks for an already-active thread that plausibly covers this
|
||||
// cluster, by matching a discriminative cluster keyword against thread titles.
|
||||
func (t *AutoThreader) findExisting(c index.TopicCluster) int64 {
|
||||
for _, kw := range c.Keywords {
|
||||
id, err := t.DB.ActiveThreadByKeyword(kw)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
if id == 0 {
|
||||
continue
|
||||
}
|
||||
th, err := t.DB.GetThread(id)
|
||||
if err != nil || th == nil {
|
||||
continue
|
||||
}
|
||||
titleTokens := toSet(strings.ToLower(th.Title))
|
||||
for _, kw2 := range c.Keywords {
|
||||
if titleTokens[kw2] {
|
||||
return id
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (t *AutoThreader) title(c index.TopicCluster) string {
|
||||
return c.Name
|
||||
}
|
||||
|
||||
func (t *AutoThreader) tags(c index.TopicCluster) string {
|
||||
var keep []string
|
||||
for _, kw := range c.Keywords {
|
||||
if len(keep) >= 5 {
|
||||
break
|
||||
}
|
||||
keep = append(keep, kw)
|
||||
}
|
||||
return strings.Join(keep, " ")
|
||||
}
|
||||
|
||||
func (t *AutoThreader) priority(c index.TopicCluster) string {
|
||||
srcs := len(sourceList(c.Entries))
|
||||
if srcs >= 3 {
|
||||
return "high"
|
||||
}
|
||||
return "medium"
|
||||
}
|
||||
|
||||
// motivation synthesizes "why" from the cluster's strongest members: project
|
||||
// context, source spread, and a couple of representative titles/summaries.
|
||||
func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string {
|
||||
sources := sourceList(c.Entries)
|
||||
|
||||
proj := dominantProject(c.Entries)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", "))
|
||||
if proj != "" {
|
||||
fmt.Fprintf(&b, " Dominant project: %s.", proj)
|
||||
}
|
||||
fmt.Fprintf(&b, " Keywords: %s.", c.Name)
|
||||
fmt.Fprintf(&b, " Representative observations:")
|
||||
|
||||
shown := 0
|
||||
seen := make(map[string]bool)
|
||||
for _, e := range c.Entries {
|
||||
if shown >= 3 {
|
||||
break
|
||||
}
|
||||
txt := strings.TrimSpace(e.Title)
|
||||
if txt == "" {
|
||||
txt = strings.TrimSpace(e.Summary)
|
||||
}
|
||||
if txt == "" || seen[txt] {
|
||||
continue
|
||||
}
|
||||
seen[txt] = true
|
||||
fmt.Fprintf(&b, " • %s", truncate(txt, 90))
|
||||
shown++
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// --- small helpers ---------------------------------------------------------
|
||||
|
||||
func fingerprints(entries []db.Entry) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.Fingerprint == "" || seen[e.Fingerprint] {
|
||||
continue
|
||||
}
|
||||
seen[e.Fingerprint] = true
|
||||
out = append(out, e.Fingerprint)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sourceList(entries []db.Entry) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.SourceID == "" || seen[e.SourceID] {
|
||||
continue
|
||||
}
|
||||
seen[e.SourceID] = true
|
||||
out = append(out, e.SourceID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dominantProject(entries []db.Entry) string {
|
||||
counts := make(map[string]int)
|
||||
for _, e := range entries {
|
||||
if e.Project != "" {
|
||||
counts[e.Project]++
|
||||
}
|
||||
}
|
||||
best, n := "", 0
|
||||
for p, c := range counts {
|
||||
if c > n {
|
||||
best, n = p, c
|
||||
}
|
||||
}
|
||||
if n*2 >= len(entries) {
|
||||
return best
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func toSet(s string) map[string]bool {
|
||||
set := make(map[string]bool)
|
||||
for _, w := range strings.Fields(s) {
|
||||
set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
Reference in New Issue
Block a user