package watch import ( "io/fs" "log" "net/http" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" "github.com/david/knox/internal/db" "github.com/david/knox/internal/ingest" "github.com/david/knox/internal/metrics" "github.com/fsnotify/fsnotify" ) const ( browserInterval = 5 * time.Minute gossipInterval = 1 * time.Minute ) type Watcher struct { knoxDB *db.KnoxDB dirs []string vault string debounce time.Duration fileIngesters []ingest.Ingester metrics *metrics.Metrics } // fileDebouncer schedules one ingest per path a settle-window after the last // relevant event (trailing edge): writers that emit bursts (multi-write appends, // atomic save = temp-write + rename) settle before anything is read, so a // partial file is never recorded as final state. Timers are removed once they // fire, keeping the map bounded by recently-active paths. type fileDebouncer struct { mu sync.Mutex timers map[string]*time.Timer } func newFileDebouncer() *fileDebouncer { return &fileDebouncer{timers: make(map[string]*time.Timer)} } func (d *fileDebouncer) schedule(path string, delay time.Duration, fn func(string)) { d.mu.Lock() defer d.mu.Unlock() if t, ok := d.timers[path]; ok { t.Stop() } d.timers[path] = time.AfterFunc(delay, func() { d.mu.Lock() delete(d.timers, path) d.mu.Unlock() fn(path) }) } // cancel drops any pending ingest for path (e.g. the file was deleted). func (d *fileDebouncer) cancel(path string) { d.mu.Lock() defer d.mu.Unlock() if t, ok := d.timers[path]; ok { t.Stop() delete(d.timers, path) } } func New(kdb *db.KnoxDB, dirs []string) *Watcher { // Detect Obsidian vault vault, _ := ingest.DetectObsidianVault() return &Watcher{ knoxDB: kdb, dirs: dirs, vault: vault, debounce: 2 * time.Second, fileIngesters: []ingest.Ingester{ ingest.NewSessionDiffIngester(), ingest.NewLogIngester(), ingest.NewSkillsIngester(), }, metrics: metrics.New(kdb, "knox"), } } func (w *Watcher) Start() error { watcher, err := fsnotify.NewWatcher() if err != nil { return err } defer watcher.Close() // Start the gossip server first so peers can reach us while the initial // seed is still ingesting. Vault/dirs are logged after the seed below. gossipAddr := ListenAddr() node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics} srv := &http.Server{Addr: gossipAddr, Handler: node.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second} go func() { log.Printf("[knox] gossip listening on %s", gossipAddr) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Printf("[knox] gossip server: %v", err) } }() // Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR). metricsAddr := MetricsAddr() metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second} go func() { log.Printf("[knox] metrics listening on %s", metricsAddr) if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Printf("[knox] metrics server: %v", err) } }() if peers := PeerAddrs(); len(peers) > 0 { log.Printf("[knox] gossip peers: %v", peers) } w.seed(watcher) log.Printf("[knox] watching %d directories", len(w.dirs)) if w.vault != "" { log.Printf("[knox] obsidian vault: %s", w.vault) } debounce := newFileDebouncer() browserTicker := time.NewTicker(browserInterval) giteaTicker := time.NewTicker(10 * time.Minute) gitTicker := time.NewTicker(10 * time.Minute) threadTicker := time.NewTicker(10 * time.Minute) gossipTicker := time.NewTicker(gossipInterval) var browserRunning, giteaRunning, gitRunning, threadRunning, gossipRunning atomic.Bool for { select { case event, ok := <-watcher.Events: if !ok { return nil } // fsnotify is non-recursive: files appearing inside newly created // subdirectories would otherwise be invisible to the daemon. isDir := false if event.Has(fsnotify.Create) { if info, err := os.Stat(event.Name); err == nil && info.IsDir() { isDir = true if err := watcher.Add(event.Name); err != nil { log.Printf("[knox] cannot watch new dir %s: %v", event.Name, err) } else { log.Printf("[knox] watching new dir %s", event.Name) } } } if event.Has(fsnotify.Remove) { // Drop any pending ingest for a deleted file. (No tombstone is // written yet — the entry lingers until reconcile/prune.) debounce.cancel(event.Name) continue } if isDir || !w.isRelevantEvent(event) { continue } trigger := eventOpName(event.Op) log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name)) name := event.Name debounce.schedule(name, w.debounce, func(path string) { w.ingestFile(path, trigger) }) case <-browserTicker.C: if !browserRunning.CompareAndSwap(false, true) { continue } go func() { defer browserRunning.Store(false) w.ingestBrowserHistory() }() case <-giteaTicker.C: if !giteaRunning.CompareAndSwap(false, true) { continue } go func() { defer giteaRunning.Store(false) w.ingestGitea() }() case <-gitTicker.C: if !gitRunning.CompareAndSwap(false, true) { continue } go func() { defer gitRunning.Store(false) w.ingestGit() }() case <-threadTicker.C: if !threadRunning.CompareAndSwap(false, true) { continue } go func() { defer threadRunning.Store(false) w.autoThread() }() case <-gossipTicker.C: if !gossipRunning.CompareAndSwap(false, true) { continue } go func() { defer gossipRunning.Store(false) w.syncGossip() }() case err, ok := <-watcher.Errors: if !ok { return nil } log.Printf("[knox] watch error: %v", err) } } } func (w *Watcher) seed(watcher *fsnotify.Watcher) { for _, dir := range w.dirs { if err := w.watchTree(watcher, dir); err != nil { log.Printf("[knox] cannot watch %s: %v", dir, err) } } // Watch Obsidian vault (every subdirectory, non-recursively mirrored) if w.vault != "" { if err := w.watchTree(watcher, w.vault); err != nil { log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err) } } // Seed existing files anywhere under the watched dirs. fsnotify watches the // whole tree, so live events cover any depth; this walk covers startup so // pre-existing nested files (e.g. skills at two+ levels) are indexed too. for _, dir := range w.dirs { _ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { return nil // skip unreadable entries; dirs are handled by the watch } for _, ing := range w.fileIngesters { if MatchesIngester(path, ing.SourceID()) { w.ingestFileWith(path, ing, "seed") return nil } } return nil }) } // Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian) // and covers all depths. if w.vault != "" { if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil { for _, r := range notes { w.recordResult(r, "seed") } } } } // watchTree adds a directory and every non-hidden subdirectory to the watcher, // mirroring fsnotify's non-recursive API with an explicit walk. Hidden // directories (.git, .obsidian, .trash) are skipped so their churn doesn't burn // inotify watches. func (w *Watcher) watchTree(watcher *fsnotify.Watcher, root string) error { rootAbs, err := filepath.Abs(root) if err != nil { return err } if err := watcher.Add(rootAbs); err != nil { return err } log.Printf("[knox] watching %s", rootAbs) return filepath.WalkDir(rootAbs, func(path string, d fs.DirEntry, err error) error { if err != nil { return nil } if !d.IsDir() { return nil } if path != rootAbs && strings.HasPrefix(d.Name(), ".") { return filepath.SkipDir } if path == rootAbs { return nil } if err := watcher.Add(path); err != nil { log.Printf("[knox] cannot watch %s: %v", path, err) } return nil }) } func (w *Watcher) ingestFile(path, trigger string) { for _, ing := range w.fileIngesters { if MatchesIngester(path, ing.SourceID()) { w.ingestFileWith(path, ing, trigger) return } } // Obsidian: any .md file in the vault if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") { obs := ingest.NewObsidianFileIngester(w.vault) w.ingestFileWith(path, obs, trigger) } } func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) { result, err := ing.Ingest(path) if err != nil { log.Printf("[knox] ingest error %s: %v", path, err) return } if result == nil { return } // Staleness check: skip when the signal time is known and unchanged. // Empty CreatedAt must NOT skip — it would suppress everything. if result.CreatedAt != "" { if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt { return } } w.recordResult(result, trigger) } // recordResult persists one ingest result and fires side effects (session // tracking, golden-thread linking happens inside RecordObservation). func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) { obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, LineStart: result.LineStart, LineEnd: result.LineEnd, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: trigger, Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { log.Printf("[knox] db error: %v", err) return } if isNew { log.Printf("[knox] new entry #%d: %s", obsID, result.Title) } else { log.Printf("[knox] updated entry #%d: %s", obsID, result.Title) } if result.SourceID == "opencode-session" { sessionID, _ := result.Provenance["session_id"].(string) if sessionID != "" { if err := w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, "active"); err != nil { log.Printf("[knox] session upsert error: %v", err) } } } } func (w *Watcher) ingestBrowserHistory() { log.Printf("[knox] periodic browser history ingest...") ing := ingest.NewBrowserHistoryIngester() results, err := ing.IngestAll() if err != nil { log.Printf("[knox] browser ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "browser_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped) } } func (w *Watcher) ingestGitea() { log.Printf("[knox] periodic gitea ingest...") ing := ingest.NewGiteaIngester() results, err := ing.IngestAll() if err != nil { log.Printf("[knox] gitea ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "gitea_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped) } } func (w *Watcher) ingestGit() { log.Printf("[knox] periodic git status ingest...") ing := ingest.NewGitIngester() ing.Recursive = true if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" { for _, part := range strings.Split(v, ",") { part = strings.TrimSpace(part) if part == "" { continue } ing.Roots = append(ing.Roots, part) } } results, err := ing.IngestAll() if err != nil { log.Printf("[knox] git ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "git_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped) } } // autoThread runs the heuristic auto-threader. Disabled entirely when // KNOX_THREAD_DISABLE is non-empty. func (w *Watcher) autoThread() { if os.Getenv("KNOX_THREAD_DISABLE") != "" { return } threader := NewAutoThreader(w.knoxDB) created, linked, err := threader.AutoThread() if err != nil { log.Printf("[knox] auto-thread error: %v", err) return } if created > 0 || linked > 0 { log.Printf("[knox] auto-thread: %d created, %d linked", created, linked) } } // syncGossip runs one anti-entropy sweep against configured peers and rebuilds // derived state if anything new arrived. func (w *Watcher) syncGossip() { peers := PeerAddrs() if len(peers) == 0 { return } pulled := Run(w.knoxDB, w.metrics, peers) // A pull only appends to the observation log; the entries cache and // auto-threads are derived state that reconcile rebuilds. Comparing entry // counts can never trigger this (pushes/pulls never touch entries directly), // so reconcile fires on the sweep's newly-inserted observation count. if pulled > 0 { created, linked, err := Reconcile(w.knoxDB) if err != nil { log.Printf("[knox] gossip reconcile: %v", err) return } log.Printf("[knox] gossip reconcile done: %d created, %d linked after pulling %d obs", created, linked, pulled) } } func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool { name := filepath.Base(event.Name) // Opencode session diffs and logs if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") { return true } // Obsidian markdown files if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") { return true } return false } func MatchesIngester(path, sourceID string) bool { base := filepath.Base(path) switch sourceID { case "opencode-session": return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json") case "opencode-log": return strings.HasSuffix(base, ".log") case "skills-catalog": return base == "SKILL.md" } return false } func eventOpName(op fsnotify.Op) string { switch { case op.Has(fsnotify.Create): return "inotify:CREATE" case op.Has(fsnotify.Write): return "inotify:WRITE" case op.Has(fsnotify.Chmod): return "inotify:CHMOD" case op.Has(fsnotify.Rename): return "inotify:RENAME" default: return "inotify:UNKNOWN" } }