package watch import ( "log" "net/http" "os" "path/filepath" "strings" "time" "github.com/david/knox/internal/db" "github.com/david/knox/internal/ingest" "github.com/fsnotify/fsnotify" ) const ( browserInterval = 5 * time.Minute gossipInterval = 1 * time.Minute ) type Watcher struct { knoxDB *db.KnoxDB dirs []string vault string debounce time.Duration fileIngesters []ingest.Ingester } func New(kdb *db.KnoxDB, dirs []string) *Watcher { // Detect Obsidian vault vault, _ := ingest.DetectObsidianVault() return &Watcher{ knoxDB: kdb, dirs: dirs, vault: vault, debounce: 2 * time.Second, fileIngesters: []ingest.Ingester{ ingest.NewSessionDiffIngester(), ingest.NewLogIngester(), ingest.NewSkillsIngester(), }, } } func (w *Watcher) Start() error { watcher, err := fsnotify.NewWatcher() if err != nil { return err } defer watcher.Close() w.seed(watcher) log.Printf("[knox] watching %d directories", len(w.dirs)) if w.vault != "" { log.Printf("[knox] obsidian vault: %s", w.vault) } // Gossip: serve our observation log to peers and periodically converge. gossipAddr := ListenAddr() if peers := PeerAddrs(); len(peers) > 0 { node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr} srv := &http.Server{Addr: gossipAddr, Handler: node.Handler()} go func() { log.Printf("[knox] gossip listening on %s", gossipAddr) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Printf("[knox] gossip server: %v", err) } }() log.Printf("[knox] gossip peers: %v", peers) } debounceMap := make(map[string]time.Time) browserTicker := time.NewTicker(browserInterval) giteaTicker := time.NewTicker(10 * time.Minute) gitTicker := time.NewTicker(10 * time.Minute) threadTicker := time.NewTicker(10 * time.Minute) gossipTicker := time.NewTicker(gossipInterval) browserRunning := false giteaRunning := false gitRunning := false threadRunning := false gossipRunning := false for { select { case event, ok := <-watcher.Events: if !ok { return nil } if !w.isRelevantEvent(event) { continue } now := time.Now() if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce { continue } debounceMap[event.Name] = now trigger := eventOpName(event.Op) log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name)) w.ingestFile(event.Name, trigger) case <-browserTicker.C: if browserRunning { continue } browserRunning = true go func() { defer func() { browserRunning = false }() w.ingestBrowserHistory() }() case <-giteaTicker.C: if giteaRunning { continue } giteaRunning = true go func() { defer func() { giteaRunning = false }() w.ingestGitea() }() case <-gitTicker.C: if gitRunning { continue } gitRunning = true go func() { defer func() { gitRunning = false }() w.ingestGit() }() case <-threadTicker.C: if threadRunning { continue } threadRunning = true go func() { defer func() { threadRunning = false }() w.autoThread() }() case <-gossipTicker.C: if gossipRunning { continue } gossipRunning = true go func() { defer func() { gossipRunning = false }() w.syncGossip() }() case err, ok := <-watcher.Errors: if !ok { return nil } log.Printf("[knox] watch error: %v", err) } } } func (w *Watcher) seed(watcher *fsnotify.Watcher) { for _, dir := range w.dirs { abs, _ := filepath.Abs(dir) if err := watcher.Add(abs); err != nil { log.Printf("[knox] cannot watch %s: %v", abs, err) continue } log.Printf("[knox] watching %s", abs) } // Watch Obsidian vault if w.vault != "" { if err := watcher.Add(w.vault); err != nil { log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err) } else { log.Printf("[knox] watching %s (obsidian)", w.vault) } } // Seed existing files for _, ing := range w.fileIngesters { for _, dir := range w.dirs { patterns := []string{ filepath.Join(dir, "*"), filepath.Join(dir, "*", "SKILL.md"), } for _, pattern := range patterns { entries, _ := filepath.Glob(pattern) for _, path := range entries { if !MatchesIngester(path, ing.SourceID()) { continue } w.ingestFileWith(path, ing, "seed") } } } } // Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian) // and covers all depths — glob patterns would match dot-dirs and miss depth >2. if w.vault != "" { if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil { for _, r := range notes { w.recordResult(r, "seed") } } } } func (w *Watcher) ingestFile(path, trigger string) { for _, ing := range w.fileIngesters { if MatchesIngester(path, ing.SourceID()) { w.ingestFileWith(path, ing, trigger) return } } // Obsidian: any .md file in the vault if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") { obs := ingest.NewObsidianFileIngester(w.vault) w.ingestFileWith(path, obs, trigger) } } func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) { result, err := ing.Ingest(path) if err != nil { log.Printf("[knox] ingest error %s: %v", path, err) return } if result == nil { return } // Staleness check: skip when the signal time is known and unchanged. // Empty CreatedAt must NOT skip — it would suppress everything. if result.CreatedAt != "" { if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt { return } } w.recordResult(result, trigger) } // recordResult persists one ingest result and fires side effects (session // tracking, golden-thread linking happens inside RecordObservation). func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) { obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, LineStart: result.LineStart, LineEnd: result.LineEnd, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: trigger, Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { log.Printf("[knox] db error: %v", err) return } if isNew { log.Printf("[knox] new entry #%d: %s", obsID, result.Title) } else { log.Printf("[knox] updated entry #%d: %s", obsID, result.Title) } if result.SourceID == "opencode-session" { sessionID, _ := result.Provenance["session_id"].(string) if sessionID != "" { status := "active" w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status) } } } func (w *Watcher) ingestBrowserHistory() { log.Printf("[knox] periodic browser history ingest...") ing := ingest.NewBrowserHistoryIngester() results, err := ing.IngestAll() if err != nil { log.Printf("[knox] browser ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "browser_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped) } } func (w *Watcher) ingestGitea() { log.Printf("[knox] periodic gitea ingest...") ing := ingest.NewGiteaIngester() results, err := ing.IngestAll() if err != nil { log.Printf("[knox] gitea ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "gitea_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped) } } func (w *Watcher) ingestGit() { log.Printf("[knox] periodic git status ingest...") ing := ingest.NewGitIngester() ing.Recursive = true if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" { for _, part := range strings.Split(v, ",") { part = strings.TrimSpace(part) if part == "" { continue } ing.Roots = append(ing.Roots, part) } } results, err := ing.IngestAll() if err != nil { log.Printf("[knox] git ingest error: %v", err) return } var newCount, skipped int for _, result := range results { existing, _ := w.knoxDB.FindEntry(result.Fingerprint) if existing != nil && existing.CreatedAt == result.CreatedAt { skipped++ continue } _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ Fingerprint: result.Fingerprint, SourceID: result.SourceID, SourcePath: result.SourcePath, Project: result.Project, ContentType: result.ContentType, Title: result.Title, Summary: result.Summary, CreatedAt: result.CreatedAt, Confidence: result.Confidence, IngesterVersion: result.IngesterVersion, Trigger: "git_timer", Provenance: ingest.ProvenanceJSON(result.Provenance), }) if err != nil { continue } if isNew { newCount++ } } if newCount > 0 || skipped > 0 { log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped) } } // autoThread runs the heuristic auto-threader. Disabled entirely when // KNOX_THREAD_DISABLE is non-empty. func (w *Watcher) autoThread() { if os.Getenv("KNOX_THREAD_DISABLE") != "" { return } threader := NewAutoThreader(w.knoxDB) created, linked, err := threader.AutoThread() if err != nil { log.Printf("[knox] auto-thread error: %v", err) return } if created > 0 || linked > 0 { log.Printf("[knox] auto-thread: %d created, %d linked", created, linked) } } // syncGossip runs one anti-entropy sweep against configured peers and rebuilds // derived state if anything new arrived. func (w *Watcher) syncGossip() { peers := PeerAddrs() if len(peers) == 0 { return } before := 0 if n, err := w.knoxDB.EntryCount(); err == nil { before = n } Run(w.knoxDB, peers) // If new observations arrived, reconcile to pick up entries/threads they // imply (deterministic log → derived rebuild). if after, err := w.knoxDB.EntryCount(); err == nil && after > before { created, linked, err := Reconcile(w.knoxDB) if err != nil { log.Printf("[knox] gossip reconcile: %v", err) return } log.Printf("[knox] gossip reconcile done: %d created, %d linked", created, linked) } } func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool { if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) { return false } name := filepath.Base(event.Name) // Opencode session diffs and logs if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") { return true } // Obsidian markdown files if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") { return true } return false } func MatchesIngester(path, sourceID string) bool { base := filepath.Base(path) switch sourceID { case "opencode-session": return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json") case "opencode-log": return strings.HasSuffix(base, ".log") case "skills-catalog": return base == "SKILL.md" } return false } func eventOpName(op fsnotify.Op) string { switch { case op.Has(fsnotify.Create): return "inotify:CREATE" case op.Has(fsnotify.Write): return "inotify:WRITE" case op.Has(fsnotify.Chmod): return "inotify:CHMOD" default: return "inotify:UNKNOWN" } }