Initial commit: knox knowledge index

2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
@@ -0,0 +1,447 @@
+package watch
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/david/knox/internal/db"
+	"github.com/david/knox/internal/ingest"
+	"github.com/fsnotify/fsnotify"
+)
+
+const browserInterval = 5 * time.Minute
+
+type Watcher struct {
+	knoxDB        *db.KnoxDB
+	dirs          []string
+	vault         string
+	debounce      time.Duration
+	fileIngesters []ingest.Ingester
+}
+
+func New(kdb *db.KnoxDB, dirs []string) *Watcher {
+	// Detect Obsidian vault
+	vault, _ := ingest.DetectObsidianVault()
+
+	return &Watcher{
+		knoxDB:   kdb,
+		dirs:     dirs,
+		vault:    vault,
+		debounce: 2 * time.Second,
+		fileIngesters: []ingest.Ingester{
+			ingest.NewSessionDiffIngester(),
+			ingest.NewLogIngester(),
+			ingest.NewSkillsIngester(),
+		},
+	}
+}
+
+func (w *Watcher) Start() error {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return err
+	}
+	defer watcher.Close()
+
+	w.seed(watcher)
+	log.Printf("[knox] watching %d directories", len(w.dirs))
+
+	if w.vault != "" {
+		log.Printf("[knox] obsidian vault: %s", w.vault)
+	}
+
+	debounceMap := make(map[string]time.Time)
+	browserTicker := time.NewTicker(browserInterval)
+	giteaTicker := time.NewTicker(10 * time.Minute)
+	gitTicker := time.NewTicker(10 * time.Minute)
+	threadTicker := time.NewTicker(10 * time.Minute)
+	browserRunning := false
+	giteaRunning := false
+	gitRunning := false
+	threadRunning := false
+
+	for {
+		select {
+		case event, ok := <-watcher.Events:
+			if !ok {
+				return nil
+			}
+			if !w.isRelevantEvent(event) {
+				continue
+			}
+
+			now := time.Now()
+			if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce {
+				continue
+			}
+			debounceMap[event.Name] = now
+
+			trigger := eventOpName(event.Op)
+			log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
+			w.ingestFile(event.Name, trigger)
+
+		case <-browserTicker.C:
+			if browserRunning {
+				continue
+			}
+			browserRunning = true
+			go func() {
+				defer func() { browserRunning = false }()
+				w.ingestBrowserHistory()
+			}()
+		case <-giteaTicker.C:
+			if giteaRunning {
+				continue
+			}
+			giteaRunning = true
+			go func() {
+				defer func() { giteaRunning = false }()
+				w.ingestGitea()
+			}()
+		case <-gitTicker.C:
+			if gitRunning {
+				continue
+			}
+			gitRunning = true
+			go func() {
+				defer func() { gitRunning = false }()
+				w.ingestGit()
+			}()
+		case <-threadTicker.C:
+			if threadRunning {
+				continue
+			}
+			threadRunning = true
+			go func() {
+				defer func() { threadRunning = false }()
+				w.autoThread()
+			}()
+
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return nil
+			}
+			log.Printf("[knox] watch error: %v", err)
+		}
+	}
+}
+
+func (w *Watcher) seed(watcher *fsnotify.Watcher) {
+	for _, dir := range w.dirs {
+		abs, _ := filepath.Abs(dir)
+		if err := watcher.Add(abs); err != nil {
+			log.Printf("[knox] cannot watch %s: %v", abs, err)
+			continue
+		}
+		log.Printf("[knox] watching %s", abs)
+	}
+
+	// Watch Obsidian vault
+	if w.vault != "" {
+		if err := watcher.Add(w.vault); err != nil {
+			log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
+		} else {
+			log.Printf("[knox] watching %s (obsidian)", w.vault)
+		}
+	}
+
+	// Seed existing files
+	for _, ing := range w.fileIngesters {
+		for _, dir := range w.dirs {
+			patterns := []string{
+				filepath.Join(dir, "*"),
+				filepath.Join(dir, "*", "SKILL.md"),
+			}
+			for _, pattern := range patterns {
+				entries, _ := filepath.Glob(pattern)
+				for _, path := range entries {
+					if !MatchesIngester(path, ing.SourceID()) {
+						continue
+					}
+					w.ingestFileWith(path, ing, "seed")
+				}
+			}
+		}
+	}
+
+	// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
+	// and covers all depths — glob patterns would match dot-dirs and miss depth >2.
+	if w.vault != "" {
+		if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
+			for _, r := range notes {
+				w.recordResult(r, "seed")
+			}
+		}
+	}
+}
+
+func (w *Watcher) ingestFile(path, trigger string) {
+	for _, ing := range w.fileIngesters {
+		if MatchesIngester(path, ing.SourceID()) {
+			w.ingestFileWith(path, ing, trigger)
+			return
+		}
+	}
+	// Obsidian: any .md file in the vault
+	if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
+		obs := ingest.NewObsidianFileIngester(w.vault)
+		w.ingestFileWith(path, obs, trigger)
+	}
+}
+
+func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
+	result, err := ing.Ingest(path)
+	if err != nil {
+		log.Printf("[knox] ingest error %s: %v", path, err)
+		return
+	}
+	if result == nil {
+		return
+	}
+
+	// Staleness check: skip when the signal time is known and unchanged.
+	// Empty CreatedAt must NOT skip — it would suppress everything.
+	if result.CreatedAt != "" {
+		if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
+			return
+		}
+	}
+
+	w.recordResult(result, trigger)
+}
+
+// recordResult persists one ingest result and fires side effects (session
+// tracking, golden-thread linking happens inside RecordObservation).
+func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
+	obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+		Fingerprint:     result.Fingerprint,
+		SourceID:        result.SourceID,
+		SourcePath:      result.SourcePath,
+		Project:         result.Project,
+		ContentType:     result.ContentType,
+		Title:           result.Title,
+		Summary:         result.Summary,
+		CreatedAt:       result.CreatedAt,
+		LineStart:       result.LineStart,
+		LineEnd:         result.LineEnd,
+		Confidence:      result.Confidence,
+		IngesterVersion: result.IngesterVersion,
+		Trigger:         trigger,
+		Provenance:      ingest.ProvenanceJSON(result.Provenance),
+	})
+	if err != nil {
+		log.Printf("[knox] db error: %v", err)
+		return
+	}
+
+	if isNew {
+		log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
+	} else {
+		log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
+	}
+
+	if result.SourceID == "opencode-session" {
+		sessionID, _ := result.Provenance["session_id"].(string)
+		if sessionID != "" {
+			status := "active"
+			w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
+		}
+	}
+}
+
+func (w *Watcher) ingestBrowserHistory() {
+	log.Printf("[knox] periodic browser history ingest...")
+	ing := ingest.NewBrowserHistoryIngester()
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] browser ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "browser_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+func (w *Watcher) ingestGitea() {
+	log.Printf("[knox] periodic gitea ingest...")
+	ing := ingest.NewGiteaIngester()
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] gitea ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			Project:         result.Project,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "gitea_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+func (w *Watcher) ingestGit() {
+	log.Printf("[knox] periodic git status ingest...")
+	ing := ingest.NewGitIngester()
+	ing.Recursive = true
+	if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
+		for _, part := range strings.Split(v, ",") {
+			part = strings.TrimSpace(part)
+			if part == "" {
+				continue
+			}
+			ing.Roots = append(ing.Roots, part)
+		}
+	}
+	results, err := ing.IngestAll()
+	if err != nil {
+		log.Printf("[knox] git ingest error: %v", err)
+		return
+	}
+
+	var newCount, skipped int
+	for _, result := range results {
+		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
+		if existing != nil && existing.CreatedAt == result.CreatedAt {
+			skipped++
+			continue
+		}
+		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
+			Fingerprint:     result.Fingerprint,
+			SourceID:        result.SourceID,
+			SourcePath:      result.SourcePath,
+			Project:         result.Project,
+			ContentType:     result.ContentType,
+			Title:           result.Title,
+			Summary:         result.Summary,
+			CreatedAt:       result.CreatedAt,
+			Confidence:      result.Confidence,
+			IngesterVersion: result.IngesterVersion,
+			Trigger:         "git_timer",
+			Provenance:      ingest.ProvenanceJSON(result.Provenance),
+		})
+		if err != nil {
+			continue
+		}
+		if isNew {
+			newCount++
+		}
+	}
+	if newCount > 0 || skipped > 0 {
+		log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
+	}
+}
+
+// autoThread runs the heuristic auto-threader. Disabled entirely when
+// KNOX_THREAD_DISABLE is non-empty.
+func (w *Watcher) autoThread() {
+	if os.Getenv("KNOX_THREAD_DISABLE") != "" {
+		return
+	}
+	threader := NewAutoThreader(w.knoxDB)
+	created, linked, err := threader.AutoThread()
+	if err != nil {
+		log.Printf("[knox] auto-thread error: %v", err)
+		return
+	}
+	if created > 0 || linked > 0 {
+		log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
+	}
+}
+
+func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
+	if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
+		return false
+	}
+	name := filepath.Base(event.Name)
+	// Opencode session diffs and logs
+	if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
+		return true
+	}
+	// Obsidian markdown files
+	if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
+		return true
+	}
+	return false
+}
+
+func MatchesIngester(path, sourceID string) bool {
+	base := filepath.Base(path)
+	switch sourceID {
+	case "opencode-session":
+		return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
+	case "opencode-log":
+		return strings.HasSuffix(base, ".log")
+	case "skills-catalog":
+		return base == "SKILL.md"
+	}
+	return false
+}
+
+func eventOpName(op fsnotify.Op) string {
+	switch {
+	case op.Has(fsnotify.Create):
+		return "inotify:CREATE"
+	case op.Has(fsnotify.Write):
+		return "inotify:WRITE"
+	case op.Has(fsnotify.Chmod):
+		return "inotify:CHMOD"
+	default:
+		return "inotify:UNKNOWN"
+	}
+}