knox/internal/watch/watch.go

package watch

import (
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/david/knox/internal/db"
	"github.com/david/knox/internal/ingest"
	"github.com/david/knox/internal/metrics"
	"github.com/fsnotify/fsnotify"
)

const (
	browserInterval = 5 * time.Minute
	gossipInterval  = 1 * time.Minute
)

type Watcher struct {
	knoxDB        *db.KnoxDB
	dirs          []string
	vault         string
	debounce      time.Duration
	fileIngesters []ingest.Ingester
	metrics       *metrics.Metrics
}

func New(kdb *db.KnoxDB, dirs []string) *Watcher {
	// Detect Obsidian vault
	vault, _ := ingest.DetectObsidianVault()

	return &Watcher{
		knoxDB:   kdb,
		dirs:     dirs,
		vault:    vault,
		debounce: 2 * time.Second,
		fileIngesters: []ingest.Ingester{
			ingest.NewSessionDiffIngester(),
			ingest.NewLogIngester(),
			ingest.NewSkillsIngester(),
		},
		metrics: metrics.New(kdb, "knox"),
	}
}

func (w *Watcher) Start() error {
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	// Start the gossip server first so peers can reach us while the initial
	// seed is still ingesting. Vault/dirs are logged after the seed below.
	gossipAddr := ListenAddr()
	node := &Node{Kdb: w.knoxDB, Name: "knox", Addr: gossipAddr, Metrics: w.metrics}
	srv := &http.Server{Addr: gossipAddr, Handler: node.Handler()}
	go func() {
		log.Printf("[knox] gossip listening on %s", gossipAddr)
		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Printf("[knox] gossip server: %v", err)
		}
	}()

	// Prometheus scraping on a dedicated port (KNOX_METRICS_ADDR).
	metricsAddr := MetricsAddr()
	metricsSrv := &http.Server{Addr: metricsAddr, Handler: node.Metrics.Handler()}
	go func() {
		log.Printf("[knox] metrics listening on %s", metricsAddr)
		if err := metricsSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
			log.Printf("[knox] metrics server: %v", err)
		}
	}()

	if peers := PeerAddrs(); len(peers) > 0 {
		log.Printf("[knox] gossip peers: %v", peers)
	}

	w.seed(watcher)
	log.Printf("[knox] watching %d directories", len(w.dirs))

	if w.vault != "" {
		log.Printf("[knox] obsidian vault: %s", w.vault)
	}

	debounceMap := make(map[string]time.Time)
	browserTicker := time.NewTicker(browserInterval)
	giteaTicker := time.NewTicker(10 * time.Minute)
	gitTicker := time.NewTicker(10 * time.Minute)
	threadTicker := time.NewTicker(10 * time.Minute)
	gossipTicker := time.NewTicker(gossipInterval)
	browserRunning := false
	giteaRunning := false
	gitRunning := false
	threadRunning := false
	gossipRunning := false

	for {
		select {
		case event, ok := <-watcher.Events:
			if !ok {
				return nil
			}
			if !w.isRelevantEvent(event) {
				continue
			}

			now := time.Now()
			if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce {
				continue
			}
			debounceMap[event.Name] = now

			trigger := eventOpName(event.Op)
			log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
			w.ingestFile(event.Name, trigger)

		case <-browserTicker.C:
			if browserRunning {
				continue
			}
			browserRunning = true
			go func() {
				defer func() { browserRunning = false }()
				w.ingestBrowserHistory()
			}()
		case <-giteaTicker.C:
			if giteaRunning {
				continue
			}
			giteaRunning = true
			go func() {
				defer func() { giteaRunning = false }()
				w.ingestGitea()
			}()
		case <-gitTicker.C:
			if gitRunning {
				continue
			}
			gitRunning = true
			go func() {
				defer func() { gitRunning = false }()
				w.ingestGit()
			}()
		case <-threadTicker.C:
			if threadRunning {
				continue
			}
			threadRunning = true
			go func() {
				defer func() { threadRunning = false }()
				w.autoThread()
			}()
		case <-gossipTicker.C:
			if gossipRunning {
				continue
			}
			gossipRunning = true
			go func() {
				defer func() { gossipRunning = false }()
				w.syncGossip()
			}()

		case err, ok := <-watcher.Errors:
			if !ok {
				return nil
			}
			log.Printf("[knox] watch error: %v", err)
		}
	}
}

func (w *Watcher) seed(watcher *fsnotify.Watcher) {
	for _, dir := range w.dirs {
		abs, _ := filepath.Abs(dir)
		if err := watcher.Add(abs); err != nil {
			log.Printf("[knox] cannot watch %s: %v", abs, err)
			continue
		}
		log.Printf("[knox] watching %s", abs)
	}

	// Watch Obsidian vault
	if w.vault != "" {
		if err := watcher.Add(w.vault); err != nil {
			log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
		} else {
			log.Printf("[knox] watching %s (obsidian)", w.vault)
		}
	}

	// Seed existing files
	for _, ing := range w.fileIngesters {
		for _, dir := range w.dirs {
			patterns := []string{
				filepath.Join(dir, "*"),
				filepath.Join(dir, "*", "SKILL.md"),
			}
			for _, pattern := range patterns {
				entries, _ := filepath.Glob(pattern)
				for _, path := range entries {
					if !MatchesIngester(path, ing.SourceID()) {
						continue
					}
					w.ingestFileWith(path, ing, "seed")
				}
			}
		}
	}

	// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
	// and covers all depths — glob patterns would match dot-dirs and miss depth >2.
	if w.vault != "" {
		if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
			for _, r := range notes {
				w.recordResult(r, "seed")
			}
		}
	}
}

func (w *Watcher) ingestFile(path, trigger string) {
	for _, ing := range w.fileIngesters {
		if MatchesIngester(path, ing.SourceID()) {
			w.ingestFileWith(path, ing, trigger)
			return
		}
	}
	// Obsidian: any .md file in the vault
	if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
		obs := ingest.NewObsidianFileIngester(w.vault)
		w.ingestFileWith(path, obs, trigger)
	}
}

func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
	result, err := ing.Ingest(path)
	if err != nil {
		log.Printf("[knox] ingest error %s: %v", path, err)
		return
	}
	if result == nil {
		return
	}

	// Staleness check: skip when the signal time is known and unchanged.
	// Empty CreatedAt must NOT skip — it would suppress everything.
	if result.CreatedAt != "" {
		if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
			return
		}
	}

	w.recordResult(result, trigger)
}

// recordResult persists one ingest result and fires side effects (session
// tracking, golden-thread linking happens inside RecordObservation).
func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
	obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
		Fingerprint:     result.Fingerprint,
		SourceID:        result.SourceID,
		SourcePath:      result.SourcePath,
		Project:         result.Project,
		ContentType:     result.ContentType,
		Title:           result.Title,
		Summary:         result.Summary,
		CreatedAt:       result.CreatedAt,
		LineStart:       result.LineStart,
		LineEnd:         result.LineEnd,
		Confidence:      result.Confidence,
		IngesterVersion: result.IngesterVersion,
		Trigger:         trigger,
		Provenance:      ingest.ProvenanceJSON(result.Provenance),
	})
	if err != nil {
		log.Printf("[knox] db error: %v", err)
		return
	}

	if isNew {
		log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
	} else {
		log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
	}

	if result.SourceID == "opencode-session" {
		sessionID, _ := result.Provenance["session_id"].(string)
		if sessionID != "" {
			status := "active"
			w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
		}
	}
}

func (w *Watcher) ingestBrowserHistory() {
	log.Printf("[knox] periodic browser history ingest...")
	ing := ingest.NewBrowserHistoryIngester()
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] browser ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "browser_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
	}
}

func (w *Watcher) ingestGitea() {
	log.Printf("[knox] periodic gitea ingest...")
	ing := ingest.NewGiteaIngester()
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] gitea ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			Project:         result.Project,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "gitea_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
	}
}

func (w *Watcher) ingestGit() {
	log.Printf("[knox] periodic git status ingest...")
	ing := ingest.NewGitIngester()
	ing.Recursive = true
	if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
		for _, part := range strings.Split(v, ",") {
			part = strings.TrimSpace(part)
			if part == "" {
				continue
			}
			ing.Roots = append(ing.Roots, part)
		}
	}
	results, err := ing.IngestAll()
	if err != nil {
		log.Printf("[knox] git ingest error: %v", err)
		return
	}

	var newCount, skipped int
	for _, result := range results {
		existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
		if existing != nil && existing.CreatedAt == result.CreatedAt {
			skipped++
			continue
		}
		_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
			Fingerprint:     result.Fingerprint,
			SourceID:        result.SourceID,
			SourcePath:      result.SourcePath,
			Project:         result.Project,
			ContentType:     result.ContentType,
			Title:           result.Title,
			Summary:         result.Summary,
			CreatedAt:       result.CreatedAt,
			Confidence:      result.Confidence,
			IngesterVersion: result.IngesterVersion,
			Trigger:         "git_timer",
			Provenance:      ingest.ProvenanceJSON(result.Provenance),
		})
		if err != nil {
			continue
		}
		if isNew {
			newCount++
		}
	}
	if newCount > 0 || skipped > 0 {
		log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
	}
}

// autoThread runs the heuristic auto-threader. Disabled entirely when
// KNOX_THREAD_DISABLE is non-empty.
func (w *Watcher) autoThread() {
	if os.Getenv("KNOX_THREAD_DISABLE") != "" {
		return
	}
	threader := NewAutoThreader(w.knoxDB)
	created, linked, err := threader.AutoThread()
	if err != nil {
		log.Printf("[knox] auto-thread error: %v", err)
		return
	}
	if created > 0 || linked > 0 {
		log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
	}
}

// syncGossip runs one anti-entropy sweep against configured peers and rebuilds
// derived state if anything new arrived.
func (w *Watcher) syncGossip() {
	peers := PeerAddrs()
	if len(peers) == 0 {
		return
	}

	before := 0
	if n, err := w.knoxDB.EntryCount(); err == nil {
		before = n
	}

	Run(w.knoxDB, w.metrics, peers)

	// If new observations arrived, reconcile to pick up entries/threads they
	// imply (deterministic log → derived rebuild).
	if after, err := w.knoxDB.EntryCount(); err == nil && after > before {
		created, linked, err := Reconcile(w.knoxDB)
		if err != nil {
			log.Printf("[knox] gossip reconcile: %v", err)
			return
		}
		log.Printf("[knox] gossip reconcile done: %d created, %d linked", created, linked)
	}
}

func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
	if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
		return false
	}
	name := filepath.Base(event.Name)
	// Opencode session diffs and logs
	if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
		return true
	}
	// Obsidian markdown files
	if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
		return true
	}
	return false
}

func MatchesIngester(path, sourceID string) bool {
	base := filepath.Base(path)
	switch sourceID {
	case "opencode-session":
		return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
	case "opencode-log":
		return strings.HasSuffix(base, ".log")
	case "skills-catalog":
		return base == "SKILL.md"
	}
	return false
}

func eventOpName(op fsnotify.Op) string {
	switch {
	case op.Has(fsnotify.Create):
		return "inotify:CREATE"
	case op.Has(fsnotify.Write):
		return "inotify:WRITE"
	case op.Has(fsnotify.Chmod):
		return "inotify:CHMOD"
	default:
		return "inotify:UNKNOWN"
	}
}