Initial commit: knox knowledge index

internal/ingest/browser.go

@@ -0,0 +1,257 @@
+package ingest
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+const (
+	chromiumHistoryPath = ".config/chromium/Default/History"
+	minHistoryPath      = ".config/Min/BrowserHistory.db"
+)
+
+// noiseURLPatterns are URLs that represent navigation noise rather than knowledge.
+var noiseURLPatterns = []string{
+	"opencode.ai",
+	"auth.opencode.ai",
+	"github.com/login/oauth/authorize",
+	"accounts.google.com",
+	"google.com/accounts",
+	"kagi.com",
+	"chatgpt.com",
+	// knox indexing its own web UI is self-referential noise
+	"localhost:8924",
+	"localhost:18925",
+	"127.0.0.1:8924",
+	"127.0.0.1:18925",
+}
+
+func noiseURL(url string) bool {
+	for _, p := range noiseURLPatterns {
+		if strings.Contains(url, p) {
+			return true
+		}
+	}
+	return false
+}
+
+type BrowserHistoryIngester struct {
+	Version string
+}
+
+func NewBrowserHistoryIngester() *BrowserHistoryIngester {
+	return &BrowserHistoryIngester{Version: "browser-history/v1"}
+}
+
+func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" }
+
+type historyRow struct {
+	URL        string
+	Title      string
+	VisitTime  time.Time
+	VisitCount int
+}
+
+func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) {
+	home, _ := os.UserHomeDir()
+	var results []*IngestResult
+
+	// Chromium
+	chromiumDB := filepath.Join(home, chromiumHistoryPath)
+	cr, err := b.ingestChromium(chromiumDB)
+	if err == nil {
+		results = append(results, cr...)
+	}
+
+	// Min
+	minDB := filepath.Join(home, minHistoryPath)
+	minResults, err := b.ingestMin(minDB)
+	if err == nil {
+		results = append(results, minResults...)
+	}
+
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) {
+	if !fileExists(path) {
+		return nil, nil
+	}
+
+	// Copy to avoid lock issues (Chromium uses WAL)
+	tmp := path + ".knox_tmp"
+	if err := copyFile(path, tmp); err != nil {
+		return nil, fmt.Errorf("copy chromium db: %w", err)
+	}
+	defer os.Remove(tmp)
+
+	db, err := sql.Open("sqlite", tmp)
+	if err != nil {
+		return nil, fmt.Errorf("open chromium copy: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(
+		`SELECT url, COALESCE(title,''), last_visit_time, visit_count
+		 FROM urls
+		 WHERE last_visit_time > 0
+		 ORDER BY last_visit_time DESC
+		 LIMIT 1000`,
+	)
+	if err != nil {
+		// Table might not exist or schema mismatch
+		return nil, fmt.Errorf("query chromium urls: %w", err)
+	}
+	defer rows.Close()
+
+	var results []*IngestResult
+	for rows.Next() {
+		var url, title string
+		var visitTimeMicro int64
+		var visitCount int
+		if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil {
+			continue
+		}
+
+		// Chromium webkit time: microseconds since 1601-01-01 UTC
+		visitTime := webkitToTime(visitTimeMicro)
+
+		if result := b.makeResult(url, title, visitTime, visitCount); result != nil {
+			results = append(results, result)
+		}
+	}
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) {
+	if !fileExists(path) {
+		return nil, nil
+	}
+
+	tmp := path + ".knox_tmp"
+	if err := copyFile(path, tmp); err != nil {
+		return nil, fmt.Errorf("copy min db: %w", err)
+	}
+	defer os.Remove(tmp)
+
+	db, err := sql.Open("sqlite", tmp)
+	if err != nil {
+		return nil, fmt.Errorf("open min copy: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(
+		`SELECT url, COALESCE(title,''), timestamp
+		 FROM history
+		 ORDER BY timestamp DESC
+		 LIMIT 1000`,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("query min history: %w", err)
+	}
+	defer rows.Close()
+
+	var results []*IngestResult
+	for rows.Next() {
+		var url, title string
+		var ts int64
+		if err := rows.Scan(&url, &title, &ts); err != nil {
+			continue
+		}
+
+		visitTime := time.Unix(ts, 0)
+		if result := b.makeResult(url, title, visitTime, 1); result != nil {
+			results = append(results, result)
+		}
+	}
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult {
+	if url == "" {
+		return nil
+	}
+
+	// Skip chrome://, about://, file://, devtools
+	if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") ||
+		strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") ||
+		strings.HasPrefix(url, "chrome-extension://") {
+		return nil
+	}
+
+	if noiseURL(url) {
+		return nil
+	}
+
+	displayTitle := title
+	if displayTitle == "" {
+		displayTitle = extractDomain(url)
+	}
+
+	fp := Fingerprint([]byte(url))
+
+	confidence := 0.6
+	if visitCount > 5 {
+		confidence = 0.9
+	} else if visitCount > 1 {
+		confidence = 0.7
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        b.SourceID(),
+		SourcePath:      url,
+		ContentType:     "url",
+		Title:           displayTitle,
+		Summary:         url,
+		CreatedAt:       visitTime.Format(time.RFC3339),
+		Confidence:      confidence,
+		IngesterVersion: b.Version,
+		Provenance: map[string]any{
+			"url":         url,
+			"visit_count": visitCount,
+			"visited_at":  visitTime.Format(time.RFC3339),
+		},
+	}
+}
+
+// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time
+func webkitToTime(micros int64) time.Time {
+	// WebKit epoch: 1601-01-01 UTC
+	// Unix epoch: 1970-01-01 UTC
+	// Difference: 11644473600 seconds
+	secs := micros / 1_000_000
+	if secs < 11644473600 {
+		return time.Time{}
+	}
+	return time.Unix(secs-11644473600, 0)
+}
+
+func extractDomain(url string) string {
+	url = strings.TrimPrefix(url, "https://")
+	url = strings.TrimPrefix(url, "http://")
+	parts := strings.Split(url, "/")
+	if len(parts) > 0 {
+		return parts[0]
+	}
+	return url
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func copyFile(src, dst string) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(dst, data, 0644)
+}

internal/ingest/git.go

@@ -0,0 +1,269 @@
+package ingest
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+// GitIngester tracks project status for local git repositories.
+//
+// For each repo under one or more scan roots (default ~/src) it records the
+// current branch, whether the working tree is clean, ahead/behind counts vs
+// the upstream, the HEAD hash, and the last commit time + subject. The entry's
+// CreatedAt is the last commit time, so staleness/dedup works naturally and
+// an entry updates when new commits land.
+//
+// When Recursive is set, repos are discovered at any depth under a root
+// (following symlinked directories), not just direct children — this picks up
+// nested repos like ~/assistant/kitmaker or ~/Arduino/*/... Repos already have
+// a stable identity via their real (symlink-resolved) path, so overlapping
+// roots across symlinks do not duplicate entries.
+type GitIngester struct {
+	Version   string
+	Root      string   // single scan root; overridden by Roots (empty => $HOME/src)
+	Roots     []string // multi-scan roots; overrides Root when non-empty
+	Recursive bool     // recurse into subdirs (and follow dir symlinks) to find nested .git
+}
+
+func NewGitIngester() *GitIngester {
+	return &GitIngester{Version: "git/v1"}
+}
+
+func (g *GitIngester) SourceID() string { return "git" }
+
+// scanRoots returns the effective list of scan roots.
+func (g *GitIngester) scanRoots() []string {
+	if len(g.Roots) > 0 {
+		return g.Roots
+	}
+	if g.Root != "" {
+		return []string{g.Root}
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return nil
+	}
+	return []string{filepath.Join(home, "src")}
+}
+
+func (g *GitIngester) IngestAll() ([]*IngestResult, error) {
+	// Dedup by real path so overlapping roots / symlinked roots don't double-add.
+	seen := make(map[string]struct{})
+	var results []*IngestResult
+	for _, root := range g.scanRoots() {
+		for _, repoPath := range collectGitRepos(root, g.Recursive) {
+			real, err := filepath.EvalSymlinks(repoPath)
+			if err != nil {
+				real = repoPath
+			}
+			if _, dup := seen[real]; dup {
+				continue
+			}
+			seen[real] = struct{}{}
+			if r := g.repoToResult(real); r != nil {
+				results = append(results, r)
+			}
+		}
+	}
+	return results, nil
+}
+
+func isGitRepo(path string) bool {
+	info, err := os.Stat(filepath.Join(path, ".git"))
+	return err == nil && info.IsDir()
+}
+
+// collectGitRepos returns paths of git repos under root. In non-recursive mode
+// only direct children are considered (root itself is not counted), matching
+// the original behavior. In recursive mode the root is walked to any depth,
+// descending into symlinked directories, up to and including the root if it is
+// itself a git repo. Descending stops once a git repo is found, so nested
+// repos (e.g. a vendored copy inside another repo) are treated as leaves.
+func collectGitRepos(root string, recursive bool) []string {
+	visited := make(map[string]struct{})
+	var out []string
+
+	var walk func(dir string)
+	walk = func(dir string) {
+		real, err := filepath.EvalSymlinks(dir)
+		if err != nil {
+			return
+		}
+		if _, ok := visited[real]; ok {
+			return
+		}
+		visited[real] = struct{}{}
+
+		if isGitRepo(real) {
+			out = append(out, real)
+			return
+		}
+
+		entries, err := os.ReadDir(real)
+		if err != nil {
+			return
+		}
+		for _, e := range entries {
+			if strings.HasPrefix(e.Name(), ".") {
+				continue
+			}
+			child := filepath.Join(real, e.Name())
+			if recursive {
+				walk(child)
+				continue
+			}
+			if e.IsDir() && isGitRepo(child) {
+				out = append(out, child)
+			}
+		}
+	}
+
+	walk(root)
+	return out
+}
+
+type GitStatus struct {
+	Branch     string
+	Head       string
+	Dirty      int
+	Ahead      int
+	Behind     int
+	LastCommit string
+	Subject    string
+}
+
+func (g *GitIngester) repoToResult(repoPath string) *IngestResult {
+	status := gitStatus(repoPath)
+	name := filepath.Base(repoPath)
+	fp := Fingerprint([]byte("git:repo:" + repoPath))
+
+	clean := "clean"
+	if status.Dirty > 0 {
+		clean = fmt.Sprintf("%d change(s)", status.Dirty)
+	}
+
+	summary := fmt.Sprintf("%s @ %s — %s", name, status.Branch, clean)
+	if status.Ahead > 0 || status.Behind > 0 {
+		summary += fmt.Sprintf(", ahead %d / behind %d", status.Ahead, status.Behind)
+	}
+	if status.Subject != "" {
+		summary += " — “" + truncate(status.Subject, 80) + "”"
+	}
+
+	createdAt := status.LastCommit
+	if createdAt == "" {
+		createdAt = gitStatusTime(status)
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        g.SourceID(),
+		SourcePath:      repoPath,
+		Project:         name,
+		ContentType:     "git-status",
+		Title:           name,
+		Summary:         summary,
+		CreatedAt:       createdAt,
+		Confidence:      0.9,
+		IngesterVersion: g.Version,
+		Provenance: map[string]any{
+			"type":        "git_status",
+			"path":        repoPath,
+			"branch":      status.Branch,
+			"head":        status.Head,
+			"dirty":       status.Dirty,
+			"ahead":       status.Ahead,
+			"behind":      status.Behind,
+			"last_commit": status.LastCommit,
+			"subject":     status.Subject,
+		},
+	}
+}
+
+func gitStatusTime(s GitStatus) string {
+	if s.LastCommit != "" {
+		return s.LastCommit
+	}
+	return time.Now().UTC().Format(time.RFC3339)
+}
+
+func gitStatus(repoPath string) GitStatus {
+	out, err := gitOut(repoPath, "status", "--porcelain")
+	if err != nil {
+		return GitStatus{}
+	}
+	dirty := 0
+	if out != "" {
+		dirty = len(strings.Split(strings.TrimRight(out, "\n"), "\n"))
+	}
+
+	branch, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD")
+	head, _ := gitOut(repoPath, "rev-parse", "--short", "HEAD")
+	lastCommit, _ := gitOut(repoPath, "log", "-1", "--format=%cI")
+	subject, _ := gitOut(repoPath, "log", "-1", "--format=%s")
+
+	ahead, behind := 0, 0
+	// ahead/behind only meaningful if there's an upstream
+	if upOut, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD@{upstream}"); upOut != "" {
+		if rev, err := gitOut(repoPath, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err == nil {
+			fields := strings.Fields(rev)
+			if len(fields) == 2 {
+				var a, b int
+				if _, e1 := fmt.Sscanf(fields[0], "%d", &a); e1 == nil {
+					ahead = a
+				}
+				if _, e2 := fmt.Sscanf(fields[1], "%d", &b); e2 == nil {
+					behind = b
+				}
+			}
+		}
+	}
+
+	return GitStatus{
+		Branch:     strings.TrimSpace(branch),
+		Head:       strings.TrimSpace(head),
+		Dirty:      dirty,
+		Ahead:      ahead,
+		Behind:     behind,
+		LastCommit: strings.TrimSpace(lastCommit),
+		Subject:    strings.TrimSpace(subject),
+	}
+}
+
+// sanitizeGitField strips ANSI codes and control bytes from git output
+// (colour/unicode modes can inject escape sequences).
+var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
+
+func sanitizeGitField(s string) string {
+	s = ansiRe.ReplaceAllString(s, "")
+	s = strings.ReplaceAll(s, "\x1b", "")
+	return strings.TrimSpace(s)
+}
+
+// gitOut runs git in repoPath and returns trimmed stdout (or "" on error).
+func gitOut(repoPath string, args ...string) (string, error) {
+	cmd := exec.Command("git", append([]string{"-C", repoPath}, args...)...)
+	cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1")
+	out, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return sanitizeGitField(string(out)), nil
+}
+
+func (g *GitIngester) Ingest(_ string) (*IngestResult, error) {
+	return nil, fmt.Errorf("use IngestAll() for git")
+}
+
+var _ Ingester = (*GitIngester)(nil)
+
+// SortResults orders results deterministically by project name (helper for CLI output).
+func SortResults(res []*IngestResult) {
+	sort.Slice(res, func(i, j int) bool { return res[i].Project < res[j].Project })
+}

internal/ingest/gitea.go

@@ -0,0 +1,245 @@
+package ingest
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"time"
+)
+
+type GiteaIngester struct {
+	Version string
+}
+
+func NewGiteaIngester() *GiteaIngester {
+	return &GiteaIngester{Version: "gitea-tea/v1"}
+}
+
+func (g *GiteaIngester) SourceID() string { return "gitea" }
+
+func (g *GiteaIngester) IngestAll() ([]*IngestResult, error) {
+	var results []*IngestResult
+
+	repos, err := g.fetchRepos()
+	if err == nil {
+		for _, r := range repos {
+			if result := g.repoToResult(r); result != nil {
+				results = append(results, result)
+			}
+		}
+	}
+
+	issues, err := g.fetchIssues()
+	if err == nil {
+		for _, i := range issues {
+			if result := g.issueToResult(i); result != nil {
+				results = append(results, result)
+			}
+		}
+	}
+
+	pulls, err := g.fetchPulls()
+	if err == nil {
+		for _, p := range pulls {
+			if result := g.pullToResult(p); result != nil {
+				results = append(results, result)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+type teaRepo struct {
+	Owner       string `json:"owner"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	Updated     string `json:"updated"`
+}
+
+type teaIssue struct {
+	Index   string `json:"index"`
+	Title   string `json:"title"`
+	State   string `json:"state"`
+	Owner   string `json:"owner"`
+	Repo    string `json:"repo"`
+	Labels  string `json:"labels"`
+	Author  string `json:"author"`
+	Updated string `json:"updated"`
+}
+
+type teaPull struct {
+	Index   string `json:"index"`
+	Title   string `json:"title"`
+	State   string `json:"state"`
+	Owner   string `json:"owner"`
+	Repo    string `json:"repo"`
+	Updated string `json:"updated"`
+}
+
+func (g *GiteaIngester) teaJSON(args ...string) ([][]byte, error) {
+	cmd := exec.Command("tea", append(args, "--output", "json")...)
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("tea %s: %w", args[0], err)
+	}
+
+	var raw json.RawMessage
+	if err := json.Unmarshal(output, &raw); err != nil {
+		return nil, err
+	}
+
+	// Handle both array and object responses
+	var arr []json.RawMessage
+	if err := json.Unmarshal(raw, &arr); err == nil {
+		bytes := make([][]byte, len(arr))
+		for i, a := range arr {
+			bytes[i] = a
+		}
+		return bytes, nil
+	}
+
+	// Single object — wrap in array
+	return [][]byte{raw}, nil
+}
+
+func (g *GiteaIngester) fetchRepos() ([]teaRepo, error) {
+	items, err := g.teaJSON("repos", "list", "--limit", "100", "--fields", "owner,name,description,updated")
+	if err != nil {
+		return nil, err
+	}
+	var repos []teaRepo
+	for _, item := range items {
+		var r teaRepo
+		if err := json.Unmarshal(item, &r); err == nil {
+			repos = append(repos, r)
+		}
+	}
+	return repos, nil
+}
+
+func (g *GiteaIngester) fetchIssues() ([]teaIssue, error) {
+	items, err := g.teaJSON("issues", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,author,milestone,labels,owner,repo,updated")
+	if err != nil {
+		return nil, err
+	}
+	var issues []teaIssue
+	for _, item := range items {
+		var i teaIssue
+		if err := json.Unmarshal(item, &i); err == nil {
+			issues = append(issues, i)
+		}
+	}
+	return issues, nil
+}
+
+func (g *GiteaIngester) fetchPulls() ([]teaPull, error) {
+	items, err := g.teaJSON("pulls", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,owner,repo,updated")
+	if err != nil {
+		return nil, err
+	}
+	var pulls []teaPull
+	for _, item := range items {
+		var p teaPull
+		if err := json.Unmarshal(item, &p); err == nil {
+			pulls = append(pulls, p)
+		}
+	}
+	return pulls, nil
+}
+
+// signalTime returns the item's last-activity time, falling back to now.
+func signalTime(updated string) string {
+	if updated != "" {
+		if _, err := time.Parse(time.RFC3339, updated); err == nil {
+			return updated
+		}
+	}
+	return time.Now().UTC().Format(time.RFC3339)
+}
+
+func (g *GiteaIngester) repoToResult(r teaRepo) *IngestResult {
+	fullName := r.Owner + "/" + r.Name
+	fp := Fingerprint([]byte("gitea:repo:" + fullName))
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        g.SourceID(),
+		SourcePath:      fullName,
+		Project:         r.Name,
+		ContentType:     "repo",
+		Title:           fullName,
+		Summary:         r.Description,
+		Confidence:      0.9,
+		IngesterVersion: g.Version,
+		CreatedAt:       signalTime(r.Updated),
+		Provenance: map[string]any{
+			"type":        "repository",
+			"owner":       r.Owner,
+			"name":        r.Name,
+			"description": r.Description,
+		},
+	}
+}
+
+func (g *GiteaIngester) issueToResult(i teaIssue) *IngestResult {
+	title := fmt.Sprintf("#%s %s", i.Index, i.Title)
+	fp := Fingerprint([]byte(fmt.Sprintf("gitea:issue:%s/%s:%s", i.Owner, i.Repo, i.Index)))
+
+	summary := fmt.Sprintf("[%s/%s#%s] %s", i.Owner, i.Repo, i.Index, i.Title)
+	if i.Labels != "" {
+		summary += " [" + i.Labels + "]"
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        g.SourceID(),
+		SourcePath:      fmt.Sprintf("%s/%s#%s", i.Owner, i.Repo, i.Index),
+		Project:         i.Repo,
+		ContentType:     "issue",
+		Title:           title,
+		Summary:         summary,
+		Confidence:      0.85,
+		IngesterVersion: g.Version,
+		CreatedAt:       signalTime(i.Updated),
+		Provenance: map[string]any{
+			"type":   "issue",
+			"owner":  i.Owner,
+			"repo":   i.Repo,
+			"index":  i.Index,
+			"state":  i.State,
+			"labels": i.Labels,
+		},
+	}
+}
+
+func (g *GiteaIngester) pullToResult(p teaPull) *IngestResult {
+	title := fmt.Sprintf("!%s %s", p.Index, p.Title)
+	fp := Fingerprint([]byte(fmt.Sprintf("gitea:pull:%s/%s:%s", p.Owner, p.Repo, p.Index)))
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        g.SourceID(),
+		SourcePath:      fmt.Sprintf("%s/%s!%s", p.Owner, p.Repo, p.Index),
+		Project:         p.Repo,
+		ContentType:     "pull",
+		Title:           title,
+		Summary:         fmt.Sprintf("[PR %s/%s] %s", p.Owner, p.Repo, p.Title),
+		Confidence:      0.85,
+		IngesterVersion: g.Version,
+		CreatedAt:       signalTime(p.Updated),
+		Provenance: map[string]any{
+			"type":  "pull_request",
+			"owner": p.Owner,
+			"repo":  p.Repo,
+			"index": p.Index,
+			"state": p.State,
+		},
+	}
+}
+
+func (g *GiteaIngester) Ingest(_ string) (*IngestResult, error) {
+	return nil, fmt.Errorf("use IngestAll() for gitea")
+}
+
+var _ Ingester = (*GiteaIngester)(nil)

internal/ingest/ingest.go

@@ -0,0 +1,57 @@
+package ingest
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+)
+
+type IngestResult struct {
+	Fingerprint     string
+	SourceID        string
+	SourcePath      string
+	Project         string
+	ContentType     string
+	Title           string
+	Summary         string
+	CreatedAt       string
+	LineStart       int
+	LineEnd         int
+	Confidence      float64
+	IngesterVersion string
+	Provenance      map[string]any
+}
+
+type Ingester interface {
+	SourceID() string
+	Ingest(path string) (*IngestResult, error)
+}
+
+func Fingerprint(data []byte) string {
+	h := sha256.Sum256(data)
+	return fmt.Sprintf("%x", h[:16])
+}
+
+func FingerprintWithMeta(data []byte, meta map[string]string) string {
+	h := sha256.New()
+	h.Write(data)
+	enc := json.NewEncoder(h)
+	_ = enc.Encode(meta)
+	return fmt.Sprintf("%x", h.Sum(nil)[:16])
+}
+
+func ProvenanceJSON(m map[string]any) string {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return "{}"
+	}
+	return string(b)
+}
+
+func truncate(s string, n int) string {
+	runes := []rune(s)
+	if len(runes) <= n {
+		return s
+	}
+	return string(runes[:n]) + "..."
+}

internal/ingest/log.go

@@ -0,0 +1,138 @@
+package ingest
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var _ Ingester = (*LogIngester)(nil)
+
+type LogIngester struct {
+	Version string
+}
+
+func NewLogIngester() *LogIngester {
+	return &LogIngester{Version: "log-parser/v1"}
+}
+
+func (l *LogIngester) SourceID() string { return "opencode-log" }
+
+var (
+	sessionIDRE = regexp.MustCompile(`session[=_ ]?([a-zA-Z0-9_-]+)`)
+	projectIDRE = regexp.MustCompile(`project[=_ ]?([a-zA-Z0-9_.-]+)`)
+	modelRE     = regexp.MustCompile(`model[=_ ]?([a-zA-Z0-9_.-]+)`)
+	agentRE     = regexp.MustCompile(`agent[=_ ]?([a-zA-Z0-9_.-]+)`)
+	errorRE     = regexp.MustCompile(`(?i)(error|fail|exception|panic|timeout)`)
+)
+
+func (l *LogIngester) Ingest(path string) (*IngestResult, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+	defer f.Close()
+
+	var (
+		sessionIDs []string
+		models     []string
+		agents     []string
+		errors     []string
+		totalLines int
+		firstError int
+	)
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		totalLines++
+
+		if m := sessionIDRE.FindStringSubmatch(line); len(m) > 1 {
+			sessionIDs = append(sessionIDs, m[1])
+		}
+		if m := modelRE.FindStringSubmatch(line); len(m) > 1 {
+			models = append(models, m[1])
+		}
+		if m := agentRE.FindStringSubmatch(line); len(m) > 1 {
+			agents = append(agents, m[1])
+		}
+		if errorRE.MatchString(line) {
+			if len(errors) == 0 {
+				firstError = totalLines
+			}
+			errors = append(errors, truncate(line, 120))
+		}
+	}
+
+	base := filepath.Base(path)
+	fp := Fingerprint([]byte(path))
+	title := fmt.Sprintf("Log %s (%d lines)", base, totalLines)
+	createdAt := ""
+	if fi, err := os.Stat(path); err == nil {
+		createdAt = fi.ModTime().UTC().Format(time.RFC3339)
+	}
+
+	var summaryParts []string
+	if len(sessionIDs) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("sessions: %s", uniqueJoin(sessionIDs, 5)))
+	}
+	if len(models) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("models: %s", uniqueJoin(models, 3)))
+	}
+	if len(errors) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("%d errors (line %d)", len(errors), firstError))
+	}
+	summary := strings.Join(summaryParts, " | ")
+	if summary == "" {
+		summary = fmt.Sprintf("%d lines", totalLines)
+	}
+
+	confidence := 0.6
+	if len(errors) > 0 {
+		confidence = 0.8
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        l.SourceID(),
+		SourcePath:      path,
+		ContentType:     ".log",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       createdAt,
+		Confidence:      confidence,
+		IngesterVersion: l.Version,
+		LineEnd:         totalLines,
+		Provenance: map[string]any{
+			"session_ids": uniqueSlice(sessionIDs),
+			"models":      uniqueSlice(models),
+			"agents":      uniqueSlice(agents),
+			"error_count": len(errors),
+			"file_size":   fmt.Sprintf("%d lines", totalLines),
+		},
+	}, nil
+}
+
+func uniqueJoin(items []string, max int) string {
+	uniq := uniqueSlice(items)
+	if len(uniq) > max {
+		uniq = uniq[:max]
+	}
+	return strings.Join(uniq, ", ")
+}
+
+func uniqueSlice(items []string) []string {
+	seen := make(map[string]bool)
+	var uniq []string
+	for _, s := range items {
+		if !seen[s] {
+			seen[s] = true
+			uniq = append(uniq, s)
+		}
+	}
+	return uniq
+}

internal/ingest/obsidian.go

@@ -0,0 +1,267 @@
+package ingest
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+const obsidianConfigPath = ".config/obsidian/obsidian.json"
+
+type ObsidianIngester struct {
+	Version string
+	Vault   string // optional override; empty = auto-detect
+}
+
+func NewObsidianIngester(vault string) *ObsidianIngester {
+	return &ObsidianIngester{Version: "obsidian-note/v1", Vault: vault}
+}
+
+func (o *ObsidianIngester) SourceID() string { return "obsidian" }
+
+func (o *ObsidianIngester) IngestAll() ([]*IngestResult, error) {
+	vault := o.Vault
+	if vault == "" {
+		var err error
+		vault, err = detectObsidianVault()
+		if err != nil {
+			return nil, fmt.Errorf("no vault path and auto-detect failed: %w", err)
+		}
+	}
+
+	if !fileExists(vault) {
+		return nil, fmt.Errorf("vault path %s does not exist", vault)
+	}
+
+	var results []*IngestResult
+	err := filepath.Walk(vault, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil // skip errors per-file
+		}
+		if info.IsDir() {
+			dir := filepath.Base(path)
+			if strings.HasPrefix(dir, ".") || dir == "node_modules" {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if !strings.HasSuffix(path, ".md") {
+			return nil
+		}
+
+		result, err := o.ingestNote(path, vault)
+		if err != nil {
+			return nil
+		}
+		if result != nil {
+			results = append(results, result)
+		}
+		return nil
+	})
+
+	return results, err
+}
+
+func (o *ObsidianIngester) ingestNote(path, vault string) (*IngestResult, error) {
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) < 10 {
+		return nil, nil
+	}
+
+	content := string(data)
+	title := extractObsidianTitle(content, path, vault)
+	tags := extractObsidianTags(content)
+	created, modified := extractObsidianTimestamps(content, path)
+	if created == "" {
+		if fi, err := os.Stat(path); err == nil {
+			created = fi.ModTime().UTC().Format(time.RFC3339)
+		}
+	}
+	if modified == "" && created != "" {
+		modified = created
+	}
+
+	// Fingerprint by content (so edits create updates)
+	fp := FingerprintWithMeta(data, map[string]string{
+		"source": "obsidian",
+		"path":   path,
+	})
+
+	relPath, _ := filepath.Rel(vault, path)
+	summary := extractBodyPreview(content, 200)
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        o.SourceID(),
+		SourcePath:      relPath,
+		Project:         filepath.Base(vault),
+		ContentType:     ".md",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       created,
+		LineStart:       0,
+		LineEnd:         len(strings.Split(content, "\n")),
+		Confidence:      0.85,
+		IngesterVersion: o.Version,
+		Provenance: map[string]any{
+			"path":        relPath,
+			"tags":        tags,
+			"modified_at": modified,
+			"note_title":  title,
+		},
+	}, nil
+}
+
+func DetectObsidianVault() (string, error) { return detectObsidianVault() }
+func detectObsidianVault() (string, error) {
+	home, _ := os.UserHomeDir()
+	cfgPath := filepath.Join(home, obsidianConfigPath)
+	if !fileExists(cfgPath) {
+		// Fallback: check common vault locations
+		common := []string{
+			filepath.Join(home, "notes"),
+			filepath.Join(home, "Documents", "notes"),
+			filepath.Join(home, "Obsidian"),
+			filepath.Join(home, "vault"),
+		}
+		for _, p := range common {
+			if fileExists(filepath.Join(p, ".obsidian")) {
+				return p, nil
+			}
+		}
+		return "", fmt.Errorf("no obsidian config found at %s and no common vault detected", cfgPath)
+	}
+
+	data, err := os.ReadFile(cfgPath)
+	if err != nil {
+		return "", err
+	}
+
+	var cfg struct {
+		Vaults map[string]struct {
+			Path string `json:"path"`
+		} `json:"vaults"`
+	}
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return "", fmt.Errorf("parse obsidian config: %w", err)
+	}
+
+	for _, v := range cfg.Vaults {
+		if v.Path != "" {
+			return v.Path, nil
+		}
+	}
+	return "", fmt.Errorf("no vault path in obsidian config")
+}
+
+func extractObsidianTitle(content, path, vault string) string {
+	// Try frontmatter title first
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "title:") {
+			t := strings.TrimSpace(trimmed[6:])
+			t = strings.Trim(t, "\"'")
+			if t != "" {
+				return t
+			}
+		}
+		if strings.HasPrefix(trimmed, "aliases:") {
+			break
+		}
+		if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
+			break
+		}
+	}
+
+	// Fallback: first H1
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "# ") {
+			return strings.TrimSpace(trimmed[2:])
+		}
+	}
+
+	// Fallback: filename
+	rel, _ := filepath.Rel(vault, path)
+	return strings.TrimSuffix(rel, ".md")
+}
+
+func extractObsidianTags(content string) []string {
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "tags:") {
+			raw := strings.TrimSpace(trimmed[5:])
+			raw = strings.Trim(raw, "\"[] ")
+			var tags []string
+			for _, t := range strings.Split(raw, ",") {
+				t = strings.TrimSpace(t)
+				t = strings.Trim(t, "\" ")
+				if t != "" {
+					tags = append(tags, t)
+				}
+			}
+			return tags
+		}
+		if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
+			break
+		}
+	}
+	return nil
+}
+
+func extractObsidianTimestamps(content, path string) (created, modified string) {
+	// Try frontmatter dates
+	inFrontmatter := false
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "---" {
+			inFrontmatter = !inFrontmatter
+			continue
+		}
+		if !inFrontmatter {
+			break
+		}
+		if strings.HasPrefix(trimmed, "created:") || strings.HasPrefix(trimmed, "date:") {
+			created = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
+			created = strings.Trim(created, "\"' ")
+		}
+		if strings.HasPrefix(trimmed, "modified:") || strings.HasPrefix(trimmed, "updated:") {
+			modified = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
+			modified = strings.Trim(modified, "\"' ")
+		}
+	}
+	return created, modified
+}
+
+func extractBodyPreview(content string, maxLen int) string {
+	inFrontmatter := false
+	var body strings.Builder
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "---" {
+			if inFrontmatter {
+				inFrontmatter = false
+				continue
+			}
+			inFrontmatter = true
+			continue
+		}
+		if inFrontmatter {
+			continue
+		}
+		if trimmed != "" && body.Len() < maxLen {
+			if body.Len() > 0 {
+				body.WriteString(" ")
+			}
+			body.WriteString(truncate(trimmed, maxLen-body.Len()))
+		}
+	}
+	result := body.String()
+	if len(result) > maxLen {
+		return result[:maxLen] + "..."
+	}
+	return result
+}

internal/ingest/obsidian_file.go

@@ -0,0 +1,69 @@
+package ingest
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var _ Ingester = (*ObsidianFileIngester)(nil)
+
+type ObsidianFileIngester struct {
+	Version string
+	Vault   string
+}
+
+func NewObsidianFileIngester(vault string) *ObsidianFileIngester {
+	return &ObsidianFileIngester{Version: "obsidian-file/v1", Vault: vault}
+}
+
+func (o *ObsidianFileIngester) SourceID() string { return "obsidian" }
+
+func (o *ObsidianFileIngester) Ingest(path string) (*IngestResult, error) {
+	if !strings.HasSuffix(path, ".md") {
+		return nil, nil
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil || len(data) < 10 {
+		return nil, nil
+	}
+
+	content := string(data)
+	title := extractObsidianTitle(content, path, o.Vault)
+	tags := extractObsidianTags(content)
+	created, _ := extractObsidianTimestamps(content, path)
+	if created == "" {
+		if fi, err := os.Stat(path); err == nil {
+			created = fi.ModTime().UTC().Format(time.RFC3339)
+		}
+	}
+
+	fp := FingerprintWithMeta(data, map[string]string{
+		"source": "obsidian",
+		"path":   path,
+	})
+
+	relPath, _ := filepath.Rel(o.Vault, path)
+	summary := extractBodyPreview(content, 200)
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        o.SourceID(),
+		SourcePath:      relPath,
+		Project:         filepath.Base(o.Vault),
+		ContentType:     ".md",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       created,
+		LineEnd:         len(strings.Split(content, "\n")),
+		Confidence:      0.85,
+		IngesterVersion: o.Version,
+		Provenance: map[string]any{
+			"path":       relPath,
+			"tags":       tags,
+			"note_title": title,
+		},
+	}, nil
+}

internal/ingest/session.go

@@ -0,0 +1,157 @@
+package ingest
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type jsonKind int
+
+const (
+	jsonObject jsonKind = iota
+	jsonArray
+)
+
+var _ Ingester = (*SessionDiffIngester)(nil)
+
+type SessionDiffIngester struct {
+	Version string
+}
+
+func NewSessionDiffIngester() *SessionDiffIngester {
+	return &SessionDiffIngester{Version: "session-diff/v1"}
+}
+
+func (s *SessionDiffIngester) SourceID() string { return "opencode-session" }
+
+func (s *SessionDiffIngester) Ingest(path string) (*IngestResult, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read session diff: %w", err)
+	}
+	if len(data) < 10 {
+		return nil, nil
+	}
+
+	raw, kind := parseJSON(data)
+	if raw == nil {
+		return nil, nil
+	}
+
+	sessionID := guessString(raw, "session_id", "id", "sessionId")
+	project := guessString(raw, "project", "project_id", "projectPath", "projectId")
+	title := guessString(raw, "title", "name", "label")
+	createdAt := guessString(raw, "created_at", "timestamp", "date", "started_at")
+	if createdAt == "" {
+		if fi, err := os.Stat(path); err == nil {
+			createdAt = fi.ModTime().UTC().Format(time.RFC3339)
+		}
+	}
+
+	if sessionID == "" {
+		base := filepath.Base(path)
+		sessionID = strings.TrimSuffix(base, filepath.Ext(base))
+		sessionID = strings.TrimPrefix(sessionID, "ses_")
+	}
+	if title == "" {
+		title = fmt.Sprintf("Session %s", sessionID[:min(8, len(sessionID))])
+	}
+
+	summary := extractSummary(raw, kind)
+	fp := FingerprintWithMeta(data, map[string]string{
+		"session_id": sessionID,
+		"source":     "session_diff",
+	})
+
+	confidence := 0.7
+	if kind == jsonArray {
+		confidence = 0.4
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        s.SourceID(),
+		SourcePath:      path,
+		Project:         project,
+		ContentType:     ".json",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       createdAt,
+		LineStart:       0,
+		LineEnd:         0,
+		Confidence:      confidence,
+		IngesterVersion: s.Version,
+		Provenance: map[string]any{
+			"session_id":  sessionID,
+			"kind":        kindName(kind),
+			"field_count": len(raw),
+		},
+	}, nil
+}
+
+func parseJSON(data []byte) (map[string]any, jsonKind) {
+	var obj map[string]any
+	if err := json.Unmarshal(data, &obj); err == nil {
+		return obj, jsonObject
+	}
+	var arr []any
+	if err := json.Unmarshal(data, &arr); err == nil {
+		m := make(map[string]any)
+		m["_count"] = float64(len(arr))
+		m["_kind"] = "array"
+		if len(arr) > 0 {
+			if first, ok := arr[0].(map[string]any); ok {
+				for k, v := range first {
+					if str, ok := v.(string); ok && len(str) < 200 {
+						m["_sample_"+k] = str
+						break
+					}
+				}
+			}
+		}
+		return m, jsonArray
+	}
+	return nil, jsonObject
+}
+
+func extractSummary(raw map[string]any, kind jsonKind) string {
+	if msg, ok := raw["message"].(string); ok && len(msg) > 0 {
+		return truncate(msg, 200)
+	}
+	if msgs, ok := raw["messages"].([]any); ok && len(msgs) > 0 {
+		parts := make([]string, 0, len(msgs))
+		for _, m := range msgs {
+			if mm, ok := m.(map[string]any); ok {
+				if c, ok := mm["content"].(string); ok {
+					parts = append(parts, truncate(c, 100))
+				}
+			}
+		}
+		return strings.Join(parts, " | ")
+	}
+	if count, ok := raw["_count"].(float64); ok && kind == jsonArray {
+		return fmt.Sprintf("Array with %.0f entries", count)
+	}
+	b, _ := json.Marshal(raw)
+	return truncate(string(b), 200)
+}
+
+func guessString(m map[string]any, keys ...string) string {
+	for _, k := range keys {
+		if v, ok := m[k].(string); ok && v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func kindName(k jsonKind) string {
+	if k == jsonArray {
+		return "array"
+	}
+	return "object"
+}

internal/ingest/skills.go

@@ -0,0 +1,135 @@
+package ingest
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+var _ Ingester = (*SkillsIngester)(nil)
+
+type SkillsIngester struct {
+	Version string
+}
+
+func NewSkillsIngester() *SkillsIngester {
+	return &SkillsIngester{Version: "skills-index/v1"}
+}
+
+func (s *SkillsIngester) SourceID() string { return "skills-catalog" }
+
+func (s *SkillsIngester) Ingest(path string) (*IngestResult, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("read skill: %w", err)
+	}
+	if len(data) < 20 {
+		return nil, nil
+	}
+
+	content := string(data)
+	title := extractSkillName(content, path)
+	summary := extractSkillSummary(content)
+	tags := extractSkillTags(content)
+	projects := extractProjects(tags)
+	createdAt := extractCreatedAt(content)
+
+	fp := FingerprintWithMeta(data, map[string]string{
+		"source": "skills-catalog",
+		"title":  title,
+	})
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        s.SourceID(),
+		SourcePath:      path,
+		Project:         projects,
+		ContentType:     ".md",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       createdAt,
+		LineStart:       0,
+		LineEnd:         len(strings.Split(content, "\n")),
+		Confidence:      0.9,
+		IngesterVersion: s.Version,
+		Provenance: map[string]any{
+			"skill_name": title,
+			"tags":       tags,
+			"line_count": len(strings.Split(content, "\n")),
+		},
+	}, nil
+}
+
+func extractSkillName(content, path string) string {
+	base := filepath.Base(filepath.Dir(path))
+	if base != "" && base != "." && base != "/" {
+		return base
+	}
+	return strings.TrimSuffix(filepath.Base(path), ".md")
+}
+
+func extractSkillSummary(content string) string {
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "description:") {
+			desc := strings.TrimSpace(trimmed[len("description:"):])
+			if len(desc) > 200 {
+				desc = desc[:200] + "..."
+			}
+			return desc
+		}
+	}
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "# ") {
+			continue
+		}
+		if trimmed != "" && !strings.HasPrefix(trimmed, "---") {
+			return truncate(trimmed, 200)
+		}
+	}
+	return ""
+}
+
+func extractSkillTags(content string) []string {
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "tags:") {
+			raw := strings.TrimSpace(trimmed[len("tags:"):])
+			raw = strings.Trim(raw, "\"[]")
+			var tags []string
+			for _, t := range strings.Split(raw, ",") {
+				t = strings.TrimSpace(t)
+				t = strings.Trim(t, "\" ")
+				if t != "" {
+					tags = append(tags, t)
+				}
+			}
+			return tags
+		}
+	}
+	return nil
+}
+
+func extractProjects(tags []string) string {
+	for _, t := range tags {
+		if strings.HasPrefix(t, "scope:project:") {
+			return strings.TrimPrefix(t, "scope:project:")
+		}
+		if strings.HasPrefix(t, "scope:machine:") {
+			return "[machine]"
+		}
+	}
+	return ""
+}
+
+func extractCreatedAt(content string) string {
+	for _, line := range strings.Split(content, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "created_at:") || strings.HasPrefix(trimmed, "updated_at:") {
+			return strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
+		}
+	}
+	return ""
+}