knox/internal/watch/watch.go

package watch

import (
	"io/fs"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/david/knox/internal/db"
	"github.com/david/knox/internal/ingest"
	"github.com/david/knox/internal/metrics"
	"github.com/fsnotify/fsnotify"
)

const (
	browserInterval = 5 * time.Minute
	gossipInterval  = 1 * time.Minute
)

type Watcher struct {
	knoxDB        *db.KnoxDB
	dirs          []string
	vault         string
	debounce      time.Duration
	fileIngesters []ingest.Ingester
	metrics       *metrics.Metrics
}

// fileDebouncer schedules one ingest per path a settle-window after the last
// relevant event (trailing edge): writers that emit bursts (multi-write appends,
// atomic save = temp-write + rename) settle before anything is read, so a
// partial file is never recorded as final state. Timers are removed once they
// fire, keeping the map bounded by recently-active paths.
type fileDebouncer struct {
	mu     sync.Mutex
	timers map[string]*time.Timer
}

func newFileDebouncer() *fileDebouncer {
	return &fileDebouncer{timers: make(map[string]*time.Timer)}
}

func (d *fileDebouncer) schedule(path string, delay time.Duration, fn func(string)) {
	d.mu.Lock()
	defer d.mu.Unlock()
	if t, ok := d.timers[path]; ok {
		t.Stop()
	}
	d.timers[path] = time.AfterFunc(delay, func() {
		d.mu.Lock()
		delete(d.timers, path)
		d.mu.Unlock()
		fn(path)
	})
}

// cancel drops any pending ingest for path (e.g. the file was deleted).
func (d *fileDebouncer) cancel(path string) {
	d.mu.Lock()
	defer d.mu.Unlock()
	if t, ok := d.timers[path]; ok {
		t.Stop()
		delete(d.timers, path)
	}
}

func New(kdb *db.KnoxDB, dirs []string) *Watcher {
	// Detect Obsidian vault
	vault, _ := ingest.DetectObsidianVault()

	return &Watcher{
		knoxDB:   kdb,
		dirs:     dirs,
		vault:    vault,
		debounce: 2 * time.Second,
		fileIngesters: []ingest.Ingester{
			ingest.NewSessionDiffIngester(),
			ingest.NewLogIngester(),
			ingest.NewSkillsIngester(),
		},
		metrics: metrics.New(kdb, "knox"),
	}
}

func (w *Watcher) Start() error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	// Start the gossip server first so peers can reach us while the initial
	// seed is still ingesting. Vault/dirs are logged after the seed below.
	gossipAddr := ListenAddr()
	node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics}
	srv := &http.Server{Addr: gossipAddr, Handler: node.Handler()}
	go func() {
		log.Printf("[knox] gossip listening on %s", gossipAddr)
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Printf("[knox] gossip server: %v", err)
		}
	}()

	// Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR).
	metricsAddr := MetricsAddr()
	metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler()}
	go func() {
		log.Printf("[knox] metrics listening on %s", metricsAddr)
		if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Printf("[knox] metrics server: %v", err)
		}
	}()

	if peers := PeerAddrs(); len(peers) > 0 {
		log.Printf("[knox] gossip peers: %v", peers)
	}

	w.seed(watcher)
	log.Printf("[knox] watching %d directories", len(w.dirs))

	if w.vault != "" {
		log.Printf("[knox] obsidian vault: %s", w.vault)
	}

	debounce := newFileDebouncer()
	browserTicker := time.NewTicker(browserInterval)
	giteaTicker := time.NewTicker(10 * time.Minute)
	gitTicker := time.NewTicker(10 * time.Minute)
	threadTicker := time.NewTicker(10 * time.Minute)
	gossipTicker := time.NewTicker(gossipInterval)
	var browserRunning, giteaRunning, gitRunning, threadRunning, gossipRunning atomic.Bool

	for {
		select {
		case event, ok := <-watcher.Events:
			if !ok {
				return nil
			}

			// fsnotify is non-recursive: files appearing inside newly created
			// subdirectories would otherwise be invisible to the daemon.
			isDir := false
			if event.Has(fsnotify.Create) {
				if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
					isDir = true
					if err := watcher.Add(event.Name); err != nil {
						log.Printf("[knox] cannot watch new dir %s: %v", event.Name, err)
					} else {
						log.Printf("[knox] watching new dir %s", event.Name)
					}
				}
			}

			if event.Has(fsnotify.Remove) {
				// Drop any pending ingest for a deleted file. (No tombstone is
				// written yet — the entry lingers until reconcile/prune.)
				debounce.cancel(event.Name)
				continue
			}
			if isDir || !w.isRelevantEvent(event) {
				continue
			}

			trigger := eventOpName(event.Op)
			log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
			name := event.Name
			debounce.schedule(name, w.debounce, func(path string) {
				w.ingestFile(path, trigger)
			})

		case <-browserTicker.C:
			if !browserRunning.CompareAndSwap(false, true) {
				continue
			}
			go func() {
				defer browserRunning.Store(false)
				w.ingestBrowserHistory()
			}()
		case <-giteaTicker.C:
			if !giteaRunning.CompareAndSwap(false, true) {
				continue
			}
			go func() {
				defer giteaRunning.Store(false)
				w.ingestGitea()
			}()
		case <-gitTicker.C:
			if !gitRunning.CompareAndSwap(false, true) {
				continue
			}
			go func() {
				defer gitRunning.Store(false)
				w.ingestGit()
			}()
		case <-threadTicker.C:
			if !threadRunning.CompareAndSwap(false, true) {
				continue
			}
			go func() {
				defer threadRunning.Store(false)
				w.autoThread()
			}()
		case <-gossipTicker.C:
			if !gossipRunning.CompareAndSwap(false, true) {
				continue
			}
			go func() {
				defer gossipRunning.Store(false)
				w.syncGossip()
			}()

		case err, ok := <-watcher.Errors:
			if !ok {
				return nil
			}
			log.Printf("[knox] watch error: %v", err)
		}
	}
}

func (w *Watcher) seed(watcher *fsnotify.Watcher) {
	for _, dir := range w.dirs {
		if err := w.watchTree(watcher, dir); err != nil {
			log.Printf("[knox] cannot watch %s: %v", dir, err)
		}
	}

	// Watch Obsidian vault (every subdirectory, non-recursively mirrored)
	if w.vault != "" {
		if err := w.watchTree(watcher, w.vault); err != nil {
			log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
		}
	}

	// Seed existing files anywhere under the watched dirs. fsnotify watches the
	// whole tree, so live events cover any depth; this walk covers startup so
	// pre-existing nested files (e.g. skills at two+ levels) are indexed too.
	for _, dir := range w.dirs {
		_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
			if err != nil || d.IsDir() {
				return nil // skip unreadable entries; dirs are handled by the watch
			}
			for _, ing := range w.fileIngesters {
				if MatchesIngester(path, ing.SourceID()) {
					w.ingestFileWith(path, ing, "seed")
					return nil
				}
			}
			return nil
		})
	}

	// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
	// and covers all depths.
	if w.vault != "" {
		if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
			for _, r := range notes {
				w.recordResult(r, "seed")
			}
		}
	}
}

// watchTree adds a directory and every non-hidden subdirectory to the watcher,
// mirroring fsnotify's non-recursive API with an explicit walk. Hidden
// directories (.git, .obsidian, .trash) are skipped so their churn doesn't burn
// inotify watches.
func (w *Watcher) watchTree(watcher *fsnotify.Watcher, root string) error {
	rootAbs, err := filepath.Abs(root)
	if err != nil {
		return err
	}
	if err := watcher.Add(rootAbs); err != nil {
		return err
	}
	log.Printf("[knox] watching %s", rootAbs)
	return filepath.WalkDir(rootAbs, func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if !d.IsDir() {
			return nil
		}
		if path != rootAbs && strings.HasPrefix(d.Name(), ".") {
			return filepath.SkipDir
		}
		if path == rootAbs {
			return nil
		}
		if err := watcher.Add(path); err != nil {
			log.Printf("[knox] cannot watch %s: %v", path, err)
		}
		return nil
	})
}

func (w *Watcher) ingestFile(path, trigger string) {
	for _, ing := range w.fileIngesters {
		if MatchesIngester(path, ing.SourceID()) {
			w.ingestFileWith(path, ing, trigger)
			return
		}
	}
	// Obsidian: any .md file in the vault
	if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
		obs := ingest.NewObsidianFileIngester(w.vault)
		w.ingestFileWith(path, obs, trigger)
	}
}

func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
	result, err := ing.Ingest(path)
	if err != nil {
		log.Printf("[knox] ingest error %s: %v", path, err)
		return
	}
	if result == nil {
		return
	}

	// Staleness check: skip when the signal time is known and unchanged.
	// Empty CreatedAt must NOT skip — it would suppress everything.
	if result.CreatedAt != "" {
		if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
			return
		}
	}

	w.recordResult(result, trigger)
}

// recordResult persists one ingest result and fires side effects (session
// tracking, golden-thread linking happens inside RecordObservation).
func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
	obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
		Fingerprint:     result.Fingerprint,
		SourceID:        result.SourceID,
		SourcePath:      result.SourcePath,
		Project:         result.Project,
		ContentType:     result.ContentType,
		Title:           result.Title,
		Summary:         result.Summary,
		CreatedAt:       result.CreatedAt,
		LineStart:       result.LineStart,
		LineEnd:         result.LineEnd,
		Confidence:      result.Confidence,
		IngesterVersion: result.IngesterVersion,
		Trigger:         trigger,
		Provenance:      ingest.ProvenanceJSON(result.Provenance),
	})
	if err != nil {
		log.Printf("[knox] db error: %v", err)
		return
	}

	if isNew {
		log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
	} else {
		log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
	}

	if result.SourceID == "opencode-session" {
		sessionID, _ := result.Provenance["session_id"].(string)
		if sessionID != "" {
			status := "active"
			w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
		}
	}
}

func (w *Watcher) ingestBrowserHistory() {
	log.Printf("[knox] periodic browser history ingest...")
	ing := ingest.NewBrowserHistoryIngester()
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] browser ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "browser_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
	}
}

func (w *Watcher) ingestGitea() {
	log.Printf("[knox] periodic gitea ingest...")
	ing := ingest.NewGiteaIngester()
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] gitea ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			Project:         result.Project,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "gitea_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
	}
}

func (w *Watcher) ingestGit() {
	log.Printf("[knox] periodic git status ingest...")
	ing := ingest.NewGitIngester()
	ing.Recursive = true
	if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
		for _, part := range strings.Split(v, ",") {
			part = strings.TrimSpace(part)
			if part == "" {
				continue
			}
			ing.Roots = append(ing.Roots, part)
		}
	}
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] git ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			Project:         result.Project,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "git_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
	}
}

// autoThread runs the heuristic auto-threader. Disabled entirely when
// KNOX_THREAD_DISABLE is non-empty.
func (w *Watcher) autoThread() {
	if os.Getenv("KNOX_THREAD_DISABLE") != "" {
		return
	}
	threader := NewAutoThreader(w.knoxDB)
	created, linked, err := threader.AutoThread()
	if err != nil {
		log.Printf("[knox] auto-thread error: %v", err)
		return
	}
	if created > 0 || linked > 0 {
		log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
	}
}

// syncGossip runs one anti-entropy sweep against configured peers and rebuilds
// derived state if anything new arrived.
func (w *Watcher) syncGossip() {
	peers := PeerAddrs()
	if len(peers) == 0 {
		return
	}

	pulled := Run(w.knoxDB, w.metrics, peers)

	// A pull only appends to the observation log; the entries cache and
	// auto-threads are derived state that reconcile rebuilds. Comparing entry
	// counts can never trigger this (pushes/pulls never touch entries directly),
	// so reconcile fires on the sweep's newly-inserted observation count.
	if pulled > 0 {
		created, linked, err := Reconcile(w.knoxDB)
		if err != nil {
			log.Printf("[knox] gossip reconcile: %v", err)
			return
		}
		log.Printf("[knox] gossip reconcile done: %d created, %d linked after pulling %d obs", created, linked, pulled)
	}
}

func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
	name := filepath.Base(event.Name)
	// Opencode session diffs and logs
	if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
		return true
	}
	// Obsidian markdown files
	if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
		return true
	}
	return false
}

func MatchesIngester(path, sourceID string) bool {
	base := filepath.Base(path)
	switch sourceID {
	case "opencode-session":
		return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
	case "opencode-log":
		return strings.HasSuffix(base, ".log")
	case "skills-catalog":
		return base == "SKILL.md"
	}
	return false
}

func eventOpName(op fsnotify.Op) string {
	switch {
	case op.Has(fsnotify.Create):
		return "inotify:CREATE"
	case op.Has(fsnotify.Write):
		return "inotify:WRITE"
	case op.Has(fsnotify.Chmod):
		return "inotify:CHMOD"
	case op.Has(fsnotify.Rename):
		return "inotify:RENAME"
	default:
		return "inotify:UNKNOWN"
	}
}