Initial commit: knox knowledge index

2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
@@ -0,0 +1,311 @@
+package watch
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/david/knox/internal/db"
+	"github.com/david/knox/internal/index"
+)
+
+// AutoThreader promotes high-intent topic clusters into knowledge threads.
+//
+// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the
+// "intent bar" when it shows sustained, recently-observed activity from a
+// non-passive work source (git/gitea/session/obsidian) plus enough distinct
+// sources overall. Motivation and title are synthesized from the cluster's
+// strongest members — they can be refined later, but the grouping and linkage
+// are deterministic and cheap enough for the watch daemon's periodic tick.
+type AutoThreader struct {
+	DB         *db.KnoxDB
+	MinSources int           // distinct sources required (incl. one work source)
+	MinEntries int           // minimum cluster size
+	Recency    time.Duration // observations must be seen within this window
+	DryRun     bool
+}
+
+// workSources are signal sources that imply active intent (vs passive browsing
+// or an always-present skills catalog).
+var workSources = map[string]bool{
+	"git":              true,
+	"gitea":            true,
+	"opencode-session": true,
+	"obsidian":         true,
+	"filesystem":       true,
+}
+
+// NewAutoThreader reads the tuning knobs from the environment. All have
+// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off.
+func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader {
+	t := &AutoThreader{
+		DB:         kdb,
+		MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2),
+		MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3),
+		Recency:    time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour,
+	}
+	return t
+}
+
+func envInt(key string, def int) int {
+	if v, err := strconv.Atoi(os.Getenv(key)); err == nil {
+		return v
+	}
+	return def
+}
+
+// AutoThread clusters recent entries and creates/extends threads for clusters
+// crossing the intent bar. Returns the number of threads created and the total
+// observations linked.
+func (t *AutoThreader) AutoThread() (created, linked int, err error) {
+	entries, err := t.DB.RecentEntries(2000)
+	if err != nil {
+		return 0, 0, err
+	}
+	if len(entries) == 0 {
+		return 0, 0, nil
+	}
+
+	tfidf := index.BuildTFIDF(entries)
+	clusters := tfidf.Cluster(2, 30)
+	now := time.Now().UTC()
+
+	for _, c := range clusters {
+		fps := fingerprints(c.Entries)
+		if len(fps) == 0 || !t.crossesBar(c, fps, now) {
+			continue
+		}
+
+		// Fold into an existing active thread if one already covers these terms.
+		if existing := t.findExisting(c); existing > 0 {
+			if t.DryRun {
+				fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name)
+				continue
+			}
+			n, err := t.DB.AutoLinkThreadObservations(existing, fps)
+			if err != nil {
+				log.Printf("[knox] threader link err: %v", err)
+				continue
+			}
+			if n > 0 {
+				log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n)
+			}
+			linked += n
+			continue
+		}
+
+		title := t.title(c)
+		motivation := t.motivation(c, fps)
+		tags := t.tags(c)
+		priority := t.priority(c)
+		prov := map[string]any{
+			"trigger": "auto_thread",
+			"cluster": c.Name,
+			"score":   c.Score,
+			"sources": sourceList(c.Entries),
+		}
+		provJSON, _ := json.Marshal(prov)
+
+		if t.DryRun {
+			fmt.Printf("[threader] (dry-run) would create thread %q  [%s]\n", title, priority)
+			fmt.Printf("            motivation: %s\n", motivation)
+			fmt.Printf("            tags: %s\n", tags)
+			fmt.Printf("            %d obs | sources: %v\n", len(fps), prov["sources"])
+			continue
+		}
+
+		id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON))
+		if err != nil {
+			log.Printf("[knox] threader create err: %v", err)
+			continue
+		}
+		if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil {
+			linked += n
+		}
+		created++
+		log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority)
+	}
+	return created, linked, nil
+}
+
+// crossesBar decides whether a cluster reflects real intent.
+func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool {
+	if len(c.Entries) < t.MinEntries {
+		return false
+	}
+
+	bySource := make(map[string]bool)
+	hasWork := false
+	recent := false
+	for _, e := range c.Entries {
+		if e.SourceID != "" {
+			bySource[e.SourceID] = true
+			if workSources[e.SourceID] {
+				hasWork = true
+			}
+		}
+		if e.LastSeen != "" {
+			if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency {
+				recent = true
+			}
+		}
+	}
+
+	if len(bySource) < t.MinSources {
+		return false
+	}
+	if !hasWork {
+		return false
+	}
+	return recent
+}
+
+// findExisting looks for an already-active thread that plausibly covers this
+// cluster, by matching a discriminative cluster keyword against thread titles.
+func (t *AutoThreader) findExisting(c index.TopicCluster) int64 {
+	for _, kw := range c.Keywords {
+		id, err := t.DB.ActiveThreadByKeyword(kw)
+		if err != nil {
+			return 0
+		}
+		if id == 0 {
+			continue
+		}
+		th, err := t.DB.GetThread(id)
+		if err != nil || th == nil {
+			continue
+		}
+		titleTokens := toSet(strings.ToLower(th.Title))
+		for _, kw2 := range c.Keywords {
+			if titleTokens[kw2] {
+				return id
+			}
+		}
+	}
+	return 0
+}
+
+func (t *AutoThreader) title(c index.TopicCluster) string {
+	return c.Name
+}
+
+func (t *AutoThreader) tags(c index.TopicCluster) string {
+	var keep []string
+	for _, kw := range c.Keywords {
+		if len(keep) >= 5 {
+			break
+		}
+		keep = append(keep, kw)
+	}
+	return strings.Join(keep, " ")
+}
+
+func (t *AutoThreader) priority(c index.TopicCluster) string {
+	srcs := len(sourceList(c.Entries))
+	if srcs >= 3 {
+		return "high"
+	}
+	return "medium"
+}
+
+// motivation synthesizes "why" from the cluster's strongest members: project
+// context, source spread, and a couple of representative titles/summaries.
+func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string {
+	sources := sourceList(c.Entries)
+
+	proj := dominantProject(c.Entries)
+
+	var b strings.Builder
+	fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", "))
+	if proj != "" {
+		fmt.Fprintf(&b, " Dominant project: %s.", proj)
+	}
+	fmt.Fprintf(&b, " Keywords: %s.", c.Name)
+	fmt.Fprintf(&b, " Representative observations:")
+
+	shown := 0
+	seen := make(map[string]bool)
+	for _, e := range c.Entries {
+		if shown >= 3 {
+			break
+		}
+		txt := strings.TrimSpace(e.Title)
+		if txt == "" {
+			txt = strings.TrimSpace(e.Summary)
+		}
+		if txt == "" || seen[txt] {
+			continue
+		}
+		seen[txt] = true
+		fmt.Fprintf(&b, " • %s", truncate(txt, 90))
+		shown++
+	}
+	return b.String()
+}
+
+// --- small helpers ---------------------------------------------------------
+
+func fingerprints(entries []db.Entry) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for _, e := range entries {
+		if e.Fingerprint == "" || seen[e.Fingerprint] {
+			continue
+		}
+		seen[e.Fingerprint] = true
+		out = append(out, e.Fingerprint)
+	}
+	return out
+}
+
+func sourceList(entries []db.Entry) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for _, e := range entries {
+		if e.SourceID == "" || seen[e.SourceID] {
+			continue
+		}
+		seen[e.SourceID] = true
+		out = append(out, e.SourceID)
+	}
+	return out
+}
+
+func dominantProject(entries []db.Entry) string {
+	counts := make(map[string]int)
+	for _, e := range entries {
+		if e.Project != "" {
+			counts[e.Project]++
+		}
+	}
+	best, n := "", 0
+	for p, c := range counts {
+		if c > n {
+			best, n = p, c
+		}
+	}
+	if n*2 >= len(entries) {
+		return best
+	}
+	return ""
+}
+
+func toSet(s string) map[string]bool {
+	set := make(map[string]bool)
+	for _, w := range strings.Fields(s) {
+		set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true
+	}
+	return set
+}
+
+func truncate(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return string(r[:n]) + "…"
+}
@@ -0,0 +1,447 @@
+package watch
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/david/knox/internal/db"
+	"github.com/david/knox/internal/ingest"
+	"github.com/fsnotify/fsnotify"
+)
+
+const browserInterval = 5 * time.Minute
+
+type Watcher struct {
+	knoxDB        *db.KnoxDB
+	dirs          []string
+	vault         string
+	debounce      time.Duration
+	fileIngesters []ingest.Ingester
+}
+
+func New(kdb *db.KnoxDB, dirs []string) *Watcher {
+	// Detect Obsidian vault
+	vault, _ := ingest.DetectObsidianVault()
+
+	return &Watcher{
+		knoxDB:   kdb,
+		dirs:     dirs,
+		vault:    vault,
+		debounce: 2 * time.Second,
+		fileIngesters: []ingest.Ingester{
+			ingest.NewSessionDiffIngester(),
+			ingest.NewLogIngester(),
+			ingest.NewSkillsIngester(),
+		},
+	}
+}
+
+func (w *Watcher) Start() error {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+	defer watcher.Close()
+
+	w.seed(watcher)
+	log.Printf("[knox] watching %d directories", len(w.dirs))
+
+	if w.vault != "" {
+		log.Printf("[knox] obsidian vault: %s", w.vault)
+	}
+
+	debounceMap := make(map[string]time.Time)
+	browserTicker := time.NewTicker(browserInterval)
+	giteaTicker := time.NewTicker(10 * time.Minute)
+	gitTicker := time.NewTicker(10 * time.Minute)
+	threadTicker := time.NewTicker(10 * time.Minute)
+	browserRunning := false
+	giteaRunning := false
+	gitRunning := false
+	threadRunning := false
+
+	for {
+		select {
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return nil
+			}
+			if !w.isRelevantEvent(event) {
+				continue
+			}
+
+			now := time.Now()
+			if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce {
+				continue
+			}
+			debounceMap[event.Name] = now
+
+			trigger := eventOpName(event.Op)
+			log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
+			w.ingestFile(event.Name, trigger)
+
+		case <-browserTicker.C:
+			if browserRunning {
+				continue
+			}
+			browserRunning = true
+			go func() {
+				defer func() { browserRunning = false }()
+				w.ingestBrowserHistory()
+			}()
+		case <-giteaTicker.C:
+			if giteaRunning {
+				continue
+			}
+			giteaRunning = true
+			go func() {
+				defer func() { giteaRunning = false }()
+				w.ingestGitea()
+			}()
+		case <-gitTicker.C:
+			if gitRunning {
+				continue
+			}
+			gitRunning = true
+			go func() {
+				defer func() { gitRunning = false }()
+				w.ingestGit()
+			}()
+		case <-threadTicker.C:
+			if threadRunning {
+				continue
+			}
+			threadRunning = true
+			go func() {
+				defer func() { threadRunning = false }()
+				w.autoThread()
+			}()
+
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return nil
+			}
+			log.Printf("[knox] watch error: %v", err)
+		}
+	}
+}
+
+func (w *Watcher) seed(watcher *fsnotify.Watcher) {
+	for _, dir := range w.dirs {
+		abs, _ := filepath.Abs(dir)
+		if err := watcher.Add(abs); err != nil {
+			log.Printf("[knox] cannot watch %s: %v", abs, err)
+			continue
+		}
+		log.Printf("[knox] watching %s", abs)
+	}
+
+	// Watch Obsidian vault
+	if w.vault != "" {
+		if err := watcher.Add(w.vault); err != nil {
+			log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
+		} else {
+			log.Printf("[knox] watching %s (obsidian)", w.vault)
+		}
+	}
+
+	// Seed existing files
+	for _, ing := range w.fileIngesters {
+		for _, dir := range w.dirs {
+			patterns := []string{
+				filepath.Join(dir, "*"),
+				filepath.Join(dir, "*", "SKILL.md"),
+			}
+			for _, pattern := range patterns {
+				entries, _ := filepath.Glob(pattern)
+				for _, path := range entries {
+					if !MatchesIngester(path, ing.SourceID()) {
+						continue
+					}
+					w.ingestFileWith(path, ing, "seed")
+				}
+			}
+		}
+	}
+
+	// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
+	// and covers all depths — glob patterns would match dot-dirs and miss depth >2.
+	if w.vault != "" {
+		if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
+			for _, r := range notes {
+				w.recordResult(r, "seed")
+			}
+		}
+	}
+}
+
+func (w *Watcher) ingestFile(path, trigger string) {
+	for _, ing := range w.fileIngesters {
+		if MatchesIngester(path, ing.SourceID()) {
+			w.ingestFileWith(path, ing, trigger)
+			return
+		}
+	}
+	// Obsidian: any .md file in the vault
+	if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
+		obs := ingest.NewObsidianFileIngester(w.vault)
+		w.ingestFileWith(path, obs, trigger)
+	}
+}
+
+func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
+	result, err := ing.Ingest(path)
+	if err != nil {
+		log.Printf("[knox] ingest error %s: %v", path, err)
+		return
+	}
+	if result == nil {
+		return
+	}
+
+	// Staleness check: skip when the signal time is known and unchanged.
+	// Empty CreatedAt must NOT skip — it would suppress everything.
+	if result.CreatedAt != "" {
+		if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
+			return
+		}
+	}
+
+	w.recordResult(result, trigger)
+}
+
+// recordResult persists one ingest result and fires side effects (session
+// tracking, golden-thread linking happens inside RecordObservation).
+func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
+	obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+		Fingerprint:     result.Fingerprint,
+		SourceID:        result.SourceID,
+		SourcePath:      result.SourcePath,
+		Project:         result.Project,
+		ContentType:     result.ContentType,
+		Title:           result.Title,
+		Summary:         result.Summary,
+		CreatedAt:       result.CreatedAt,
+		LineStart:       result.LineStart,
+		LineEnd:         result.LineEnd,
+		Confidence:      result.Confidence,
+		IngesterVersion: result.IngesterVersion,
+		Trigger:         trigger,
+		Provenance:      ingest.ProvenanceJSON(result.Provenance),
+	})
+	if err != nil {
+		log.Printf("[knox] db error: %v", err)
+		return
+	}
+
+	if isNew {
+		log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
+	} else {
+		log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
+	}
+
+	if result.SourceID == "opencode-session" {
+		sessionID, _ := result.Provenance["session_id"].(string)
+		if sessionID != "" {
+			status := "active"
+			w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
+		}
+	}
+}
+
+func (w *Watcher) ingestBrowserHistory() {
+	log.Printf("[knox] periodic browser history ingest...")
+	ing := ingest.NewBrowserHistoryIngester()
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] browser ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "browser_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+func (w *Watcher) ingestGitea() {
+	log.Printf("[knox] periodic gitea ingest...")
+	ing := ingest.NewGiteaIngester()
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] gitea ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			Project:         result.Project,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "gitea_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+func (w *Watcher) ingestGit() {
+	log.Printf("[knox] periodic git status ingest...")
+	ing := ingest.NewGitIngester()
+	ing.Recursive = true
+	if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
+		for _, part := range strings.Split(v, ",") {
+			part = strings.TrimSpace(part)
+			if part == "" {
+				continue
+			}
+			ing.Roots = append(ing.Roots, part)
+		}
+	}
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] git ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			Project:         result.Project,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "git_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+// autoThread runs the heuristic auto-threader. Disabled entirely when
+// KNOX_THREAD_DISABLE is non-empty.
+func (w *Watcher) autoThread() {
+	if os.Getenv("KNOX_THREAD_DISABLE") != "" {
+		return
+	}
+	threader := NewAutoThreader(w.knoxDB)
+	created, linked, err := threader.AutoThread()
+	if err != nil {
+		log.Printf("[knox] auto-thread error: %v", err)
+		return
+	}
+	if created > 0 || linked > 0 {
+		log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
+	}
+}
+
+func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
+	if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
+		return false
+	}
+	name := filepath.Base(event.Name)
+	// Opencode session diffs and logs
+	if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
+		return true
+	}
+	// Obsidian markdown files
+	if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
+		return true
+	}
+	return false
+}
+
+func MatchesIngester(path, sourceID string) bool {
+	base := filepath.Base(path)
+	switch sourceID {
+	case "opencode-session":
+		return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
+	case "opencode-log":
+		return strings.HasSuffix(base, ".log")
+	case "skills-catalog":
+		return base == "SKILL.md"
+	}
+	return false
+}
+
+func eventOpName(op fsnotify.Op) string {
+	switch {
+	case op.Has(fsnotify.Create):
+		return "inotify:CREATE"
+	case op.Has(fsnotify.Write):
+		return "inotify:WRITE"
+	case op.Has(fsnotify.Chmod):
+		return "inotify:CHMOD"
+	default:
+		return "inotify:UNKNOWN"
+	}
+}