commit 8eecde18b49f59d0e6dda63e40003f474824effb Author: David Gwilliam Date: Sat Aug 29 02:52:32 2026 -0700 Initial commit: knox knowledge index diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a1710de --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/knox +/knox.bak +/knox.prev +*.db +*.db-shm +*.db-wal \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b99e3f8 --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module github.com/david/knox + +go 1.26 + +require ( + github.com/fsnotify/fsnotify v1.8.0 + github.com/mark3labs/mcp-go v0.17.0 + github.com/spf13/cobra v1.9.1 + modernc.org/sqlite v1.37.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/sys v0.33.0 // indirect + modernc.org/libc v1.65.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1358997 --- /dev/null +++ b/go.sum @@ -0,0 +1,70 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mark3labs/mcp-go v0.17.0 h1:5Ps6T7qXr7De/2QTqs9h6BKeZ/qdeUeGrgM5lPzi930= +github.com/mark3labs/mcp-go v0.17.0/go.mod h1:KmJndYv7GIgcPVwEKJjNcbhVQ+hJGJhrCCB/9xITzpE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= +modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= +modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= +modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs= +modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/cmd/browser.go b/internal/cmd/browser.go new file mode 100644 index 0000000..da44e89 --- /dev/null +++ b/internal/cmd/browser.go @@ -0,0 +1,115 @@ +package cmd + +import ( + "fmt" + "log" + "strings" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" +) + +func NewBrowserCmd(kdb *db.KnoxDB) *cobra.Command { + cmd := &cobra.Command{ + Use: "browser", + Short: "Ingest browser history from Chromium and Min into the knowledge index", + Long: `Reads browser history databases (Chromium and Min), fingerprints URLs, +and stores them as observations. Tracks what you've searched for and read. + +Run periodically to capture browsing exhaust.`, + RunE: func(c *cobra.Command, args []string) error { + ing := ingest.NewBrowserHistoryIngester() + log.Printf("[knox] ingesting browser history...") + + results, err := ing.IngestAll() + if err != nil { + return fmt.Errorf("browser ingest: %w", err) + } + + if len(results) == 0 { + fmt.Println("No browser history found (Chromium/Min not detected or empty).") + return nil + } + + var newCount, dupCount int + for _, result := range results { + _, isNew, err := kdb.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "browser_ingest", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + continue + } + if isNew { + newCount++ + } else { + dupCount++ + } + } + + fmt.Printf("Browser history: %d new, %d already seen\n", newCount, dupCount) + + // Show top domains + domains := make(map[string]int) + for _, r := range results { + domains[extractDomain(r.Summary)]++ + } + fmt.Println("\nTop domains:") + for _, d := range topDomains(domains, 10) { + fmt.Printf(" %-40s %d\n", d.Name, d.Count) + } + + return nil + }, + } + return cmd +} + +func topDomains(domains map[string]int, n int) []struct { + Name string + Count int +} { + var sorted []struct { + Name string + Count int + } + for name, count := range domains { + sorted = append(sorted, struct { + Name string + Count int + }{name, count}) + } + // Simple bubble sort for small n + for i := 0; i < len(sorted)-1; i++ { + for j := 0; j < len(sorted)-1-i; j++ { + if sorted[j].Count < sorted[j+1].Count { + sorted[j], sorted[j+1] = sorted[j+1], sorted[j] + } + } + } + if len(sorted) > n { + sorted = sorted[:n] + } + return sorted +} + +func extractDomain(url string) string { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + parts := strings.Split(url, "/") + if len(parts) > 0 { + return parts[0] + } + return "" +} diff --git a/internal/cmd/git.go b/internal/cmd/git.go new file mode 100644 index 0000000..1d62ebf --- /dev/null +++ b/internal/cmd/git.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" + "github.com/spf13/cobra" +) + +func NewGitCmd(kdb *db.KnoxDB) *cobra.Command { + var roots []string + var recursive bool + cmd := &cobra.Command{ + Args: cobra.NoArgs, + Use: "git", + Short: "Ingest local git repository status into the knowledge index", + Long: `Scans git repos under one or more root dirs (default ~/src) and records +each project's status — branch, clean/dirty, ahead/behind, HEAD, last commit. +With --recursive (default) repos are found at any depth, following symlinked +directories, so nested repos like ~/assistant/kitmaker are picked up. Use +'knox git' for a one-shot ingest, or rely on the watch daemon's git timer.`, + RunE: func(c *cobra.Command, args []string) error { + ing := &ingest.GitIngester{Roots: roots, Recursive: recursive} + log.Printf("[knox] ingesting git repo status...") + + results, err := ing.IngestAll() + if err != nil { + return fmt.Errorf("git ingest: %w", err) + } + + if len(results) == 0 { + fmt.Println("No git repos found.") + return nil + } + + ingest.SortResults(results) + var newCount int + for _, result := range results { + _, isNew, err := kdb.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "git_ingest", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + continue + } + if isNew { + newCount++ + } + fmt.Printf("%-24s %s\n", result.Project, result.Summary) + } + fmt.Printf("\nGit: %d repos (%d new)\n", len(results), newCount) + return nil + }, + } + cmd.Flags().StringSliceVarP(&roots, "root", "r", nil, "Directories to scan for git repos (default ~/src; repeat or comma-separate for multiple roots)") + cmd.Flags().BoolVar(&recursive, "recursive", true, "Recurse into subdirectories (following symlinks) to find nested git repos") + return cmd +} diff --git a/internal/cmd/gitea.go b/internal/cmd/gitea.go new file mode 100644 index 0000000..45783bb --- /dev/null +++ b/internal/cmd/gitea.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" +) + +func NewGiteaCmd(kdb *db.KnoxDB) *cobra.Command { + cmd := &cobra.Command{ + Use: "gitea", + Short: "Ingest Gitea repositories, issues, and PRs into the knowledge index", + Long: `Uses the 'tea' CLI to fetch repositories, open issues, and open +pull requests from Gitea. Fingerprints each by ID for dedup.`, + RunE: func(c *cobra.Command, args []string) error { + ing := ingest.NewGiteaIngester() + log.Printf("[knox] ingesting Gitea data via tea...") + + results, err := ing.IngestAll() + if err != nil { + return fmt.Errorf("gitea ingest: %w", err) + } + + if len(results) == 0 { + fmt.Println("No Gitea data found.") + return nil + } + + var repos, issues, pulls int + var newCount int + for _, result := range results { + _, isNew, err := kdb.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "gitea_ingest", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + continue + } + if isNew { + newCount++ + } + switch result.ContentType { + case "repo": + repos++ + case "issue": + issues++ + case "pull": + pulls++ + } + } + + fmt.Printf("Gitea: %d repos, %d issues, %d PRs (%d new)\n", repos, issues, pulls, newCount) + return nil + }, + } + return cmd +} diff --git a/internal/cmd/ingest.go b/internal/cmd/ingest.go new file mode 100644 index 0000000..5bfee3f --- /dev/null +++ b/internal/cmd/ingest.go @@ -0,0 +1,120 @@ +package cmd + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" + "github.com/david/knox/internal/watch" +) + +func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command { + var dirs []string + cmd := &cobra.Command{ + Use: "ingest", + Short: "One-time full sweep of all sources into the knowledge index", + Run: func(cmd *cobra.Command, args []string) { + if len(dirs) == 0 { + dirs = defaultWatchDirs() + } + + ingesters := []ingest.Ingester{ + ingest.NewSessionDiffIngester(), + ingest.NewLogIngester(), + ingest.NewSkillsIngester(), + } + + var totalNew, totalUpdated int + + record := func(result *ingest.IngestResult) { + _, isNew, err := kdb.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + LineStart: result.LineStart, + LineEnd: result.LineEnd, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "ingest", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + return + } + if isNew { + totalNew++ + } else { + totalUpdated++ + } + + if result.SourceID == "opencode-session" { + sessionID, _ := result.Provenance["session_id"].(string) + if sessionID != "" { + kdb.UpsertSession(sessionID, result.Project, result.Title, "active") + } + } + } + + for _, ing := range ingesters { + sid := ing.SourceID() + log.Printf("[knox] ingesting %s...", sid) + + for _, dir := range dirs { + patterns := []string{ + dir + "/*", + dir + "/*/SKILL.md", + } + for _, pattern := range patterns { + entries, _ := filepath.Glob(pattern) + for _, path := range entries { + if !watch.MatchesIngester(path, sid) { + continue + } + + result, err := ing.Ingest(path) + if err != nil || result == nil { + continue + } + record(result) + } + } + } + } + + // Non-file sources: obsidian vault, browser history, gitea + if vault, err := ingest.DetectObsidianVault(); err == nil { + log.Printf("[knox] ingesting obsidian...") + if results, err := ingest.NewObsidianIngester(vault).IngestAll(); err == nil { + for _, r := range results { + record(r) + } + } + } + log.Printf("[knox] ingesting browser-history...") + if results, err := ingest.NewBrowserHistoryIngester().IngestAll(); err == nil { + for _, r := range results { + record(r) + } + } + log.Printf("[knox] ingesting gitea...") + if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil { + for _, r := range results { + record(r) + } + } + + fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated) + }, + } + cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to scan (default: opencode storage/log + skills)") + return cmd +} diff --git a/internal/cmd/mcp.go b/internal/cmd/mcp.go new file mode 100644 index 0000000..db652b2 --- /dev/null +++ b/internal/cmd/mcp.go @@ -0,0 +1,655 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/index" + "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" +) + +type detailLevel int + +const ( + detailBrief detailLevel = iota + detailNormal + detailFull +) + +func parseDetail(s string) detailLevel { + switch strings.ToLower(s) { + case "brief": + return detailBrief + case "full": + return detailFull + default: + return detailNormal + } +} + +func NewMCPServer(kdb *db.KnoxDB) *server.MCPServer { + s := server.NewMCPServer( + "knox-knowledge", + "0.2.0", + server.WithResourceCapabilities(true, true), + server.WithPromptCapabilities(true), + server.WithLogging(), + ) + + // ─── knox_search ────────────────────────────────────────── + searchTool := mcp.NewTool("knox_search", + mcp.WithDescription("Search the knowledge index. Supports zoom levels: brief (lean), normal (default), full (verbose). Scope filters: all, skills, sessions, logs."), + mcp.WithString("query", mcp.Required(), mcp.Description("Search query")), + mcp.WithNumber("limit", mcp.Description("Max results (default 10)")), + mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")), + mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")), + ) + + s.AddTool(searchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + query, _ := req.Params.Arguments["query"].(string) + limit := clampInt(req, "limit", 10, 1, 50) + detail := parseDetail(getString(req, "detail", "normal")) + scope := getString(req, "scope", "all") + + results, err := kdb.Search(query, limit) + if err != nil { + return errorResult(err.Error()), nil + } + results = filterByScope(results, scope) + if len(results) == 0 { + return mcp.NewToolResultText(fmt.Sprintf("No results for %q in scope %q", query, scope)), nil + } + + return mcp.NewToolResultText(formatEntries(results, detail, kdb)), nil + }) + + // ─── knox_recent ─────────────────────────────────────────── + recentTool := mcp.NewTool("knox_recent", + mcp.WithDescription("Show recent knowledge entries with zoom levels: brief, normal (default), full."), + mcp.WithNumber("limit", mcp.Description("Max results (default 10)")), + mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")), + mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")), + ) + + s.AddTool(recentTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + limit := clampInt(req, "limit", 10, 1, 50) + detail := parseDetail(getString(req, "detail", "normal")) + scope := getString(req, "scope", "all") + + entries, err := kdb.RecentEntries(limit) + if err != nil { + return errorResult(err.Error()), nil + } + entries = filterByScope(entries, scope) + if len(entries) == 0 { + return mcp.NewToolResultText("No entries yet."), nil + } + + return mcp.NewToolResultText(formatEntries(entries, detail, kdb)), nil + }) + + // ─── knox_get ────────────────────────────────────────────── + getTool := mcp.NewTool("knox_get", + mcp.WithDescription("Get full detail for a specific entry by fingerprint (drill-down from search results)."), + mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint (full or first 8 chars)")), + ) + + s.AddTool(getTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + fp := getString(req, "fingerprint", "") + + entry, err := kdb.FindEntry(fp) + if err != nil { + return errorResult(err.Error()), nil + } + if entry == nil { + // Try fingerprint prefix match + entries, err := kdb.FindEntryByPrefix(fp, 5) + if err != nil { + return errorResult(err.Error()), nil + } + if len(entries) == 1 { + entry = &entries[0] + } else if len(entries) > 1 { + return mcp.NewToolResultText(fmt.Sprintf("Multiple matches for %q — use full fingerprint:\n%s", fp, formatEntriesBrief(entries))), nil + } else { + return mcp.NewToolResultText(fmt.Sprintf("No entry found for fingerprint %q", fp)), nil + } + } + + // Full detail: entry + observations + provenance chain + var b strings.Builder + b.WriteString(formatEntryFull(entry)) + + obs, _ := kdb.ObservationsByFingerprint(entry.Fingerprint, 20) + if len(obs) > 0 { + b.WriteString(fmt.Sprintf("\nObservations (%d):\n", len(obs))) + for _, o := range obs { + b.WriteString(fmt.Sprintf(" [%s] trigger=%s confidence=%.1f line=%d ver=%s\n", + o.CollectedAt, o.Trigger, o.Confidence, o.LineEnd, o.IngesterVersion)) + } + } + + chain, _ := kdb.ProvenanceChain(entry.Fingerprint, 3) + if len(chain) > 0 { + b.WriteString(fmt.Sprintf("\nProvenance links (%d):\n", len(chain))) + for _, l := range chain { + b.WriteString(fmt.Sprintf(" %s --[%s]--> %s (hop %d)\n", shortFP(l.Fingerprint), l.Relation, shortFP(entry.Fingerprint), l.HopDistance)) + } + } + + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_stats ──────────────────────────────────────────── + statsTool := mcp.NewTool("knox_stats", + mcp.WithDescription("Get knowledge index statistics"), + ) + + s.AddTool(statsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + stats, err := kdb.Stats() + if err != nil { + return errorResult(err.Error()), nil + } + b, _ := json.MarshalIndent(stats, "", " ") + return mcp.NewToolResultText(string(b)), nil + }) + + // ─── knox_count ──────────────────────────────────────────── + countTool := mcp.NewTool("knox_count", + mcp.WithDescription("Return hit count matching a query, grouped by source. Lightweight — no result bodies, minimal tokens."), + mcp.WithString("query", mcp.Required(), mcp.Description("Search query")), + ) + + s.AddTool(countTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + query, _ := req.Params.Arguments["query"].(string) + results, err := kdb.Search(query, 500) + if err != nil { + return errorResult(err.Error()), nil + } + total := len(results) + bySource := make(map[string]int) + for _, r := range results { + bySource[r.SourceID]++ + } + var b strings.Builder + b.WriteString(fmt.Sprintf("Matches for %q: %d total\n", query, total)) + for src, count := range bySource { + b.WriteString(fmt.Sprintf(" %-20s %d\n", src+":", count)) + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_reflect ────────────────────────────────────────── + reflectTool := mcp.NewTool("knox_reflect", + mcp.WithDescription("Process pending sessions for reflection and gap analysis"), + ) + + s.AddTool(reflectTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + reflector := index.NewReflector(kdb) + results, err := reflector.Reflect() + if err != nil { + return errorResult(err.Error()), nil + } + if len(results) == 0 { + return mcp.NewToolResultText("No pending sessions to reflect on."), nil + } + return mcp.NewToolResultText(reflector.FormatResult(results)), nil + }) + + // ─── knox_thread_list ─────────────────────────────────────── + threadListTool := mcp.NewTool("knox_thread_list", + mcp.WithDescription("List knowledge development threads with their motivation, status, and linked entry count."), + mcp.WithString("status", mcp.Description("Filter: active, stalled, resolved (default all)")), + ) + + s.AddTool(threadListTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + status := getString(req, "status", "") + threads, err := kdb.ListThreads(status) + if err != nil { + return errorResult(err.Error()), nil + } + if len(threads) == 0 { + return mcp.NewToolResultText("No threads."), nil + } + var b strings.Builder + for _, t := range threads { + b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount)) + if t.Motivation != "" { + b.WriteString(fmt.Sprintf(" %s\n", t.Motivation)) + } + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_thread_create ───────────────────────────────────── + threadCreateTool := mcp.NewTool("knox_thread_create", + mcp.WithDescription("Create a new knowledge development thread to track intent/motivation behind observations."), + mcp.WithString("title", mcp.Required(), mcp.Description("Thread title")), + mcp.WithString("motivation", mcp.Description("Why this thread exists (the 'so what')")), + mcp.WithString("priority", mcp.Description("high, medium, or low (default medium)")), + mcp.WithString("provenance", mcp.Description("JSON provenance metadata: {\"trigger\":\"...\",\"session_id\":\"...\"}")), + ) + + s.AddTool(threadCreateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + title := getString(req, "title", "") + motivation := getString(req, "motivation", "") + priority := getString(req, "priority", "medium") + provenance := getString(req, "provenance", "{}") + id, err := kdb.CreateThread(title, motivation, priority, "[]", provenance) + if err != nil { + return errorResult(err.Error()), nil + } + return mcp.NewToolResultText(fmt.Sprintf("Created thread #%d: %s", id, title)), nil + }) + + // ─── knox_thread_update ───────────────────────────────────── + threadUpdateTool := mcp.NewTool("knox_thread_update", + mcp.WithDescription("Rewrite a thread's editorial fields (title/motivation/priority/tags). Used to polish auto-generated threads."), + mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID to update")), + mcp.WithString("title", mcp.Description("New title (leave empty to keep)")), + mcp.WithString("motivation", mcp.Description("New motivation (leave empty to keep)")), + mcp.WithString("priority", mcp.Description("New priority: high/medium/low (empty to keep)")), + mcp.WithString("tags", mcp.Description("New space-separated tags (empty to keep)")), + ) + + s.AddTool(threadUpdateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + id := int64(clampInt(req, "thread_id", 0, 1, 999999)) + title := getString(req, "title", "") + motivation := getString(req, "motivation", "") + priority := getString(req, "priority", "") + tags := getString(req, "tags", "") + changed, err := kdb.UpdateThread(id, title, motivation, priority, tags) + if err != nil { + return errorResult(err.Error()), nil + } + if !changed { + return mcp.NewToolResultText(fmt.Sprintf("Thread #%d unchanged (no edits differed).", id)), nil + } + return mcp.NewToolResultText(fmt.Sprintf("Updated thread #%d", id)), nil + }) + + // ─── knox_thread_draft ────────────────────────────────────── + // Emits auto-threaded candidates with enough context (sources, linked + // observations) for an agent to write tight titles/motivations. + threadDraftTool := mcp.NewTool("knox_thread_draft", + mcp.WithDescription("List auto-threaded candidates needing LLM polish. Returns each heuristic thread's current title/motivation and its linked observations so you can rewrite them via knox_thread_update."), + mcp.WithNumber("limit", mcp.Description("Max candidates to return (default 20)")), + ) + + s.AddTool(threadDraftTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + limit := clampInt(req, "limit", 20, 1, 200) + threads, err := kdb.ListThreads("") + if err != nil { + return errorResult(err.Error()), nil + } + selected := make([]db.Thread, 0, len(threads)) + for _, t := range threads { + if strings.Contains(t.Provenance, "auto_thread") || strings.HasPrefix(t.Motivation, "Auto-detected") { + selected = append(selected, t) + } + if len(selected) >= limit { + break + } + } + if len(selected) == 0 { + return mcp.NewToolResultText("No auto-threaded candidates. Run `knox thread auto` or wait for the watch tick first."), nil + } + + var b strings.Builder + for _, t := range selected { + obs, _ := kdb.ThreadObservations(t.ID) + fmt.Fprintf(&b, "THREAD #%d [%s] %d obs\n title: %s\n motivation: %s\n", + t.ID, t.Priority, t.EntryCount, t.Title, t.Motivation) + if t.Tags != "[]" { + fmt.Fprintf(&b, " tags: %s\n", t.Tags) + } + b.WriteString(" linked observations:\n") + for _, o := range obs { + line := strings.TrimSpace(o.Title) + if o.Summary != "" { + line += " — " + strings.TrimSpace(o.Summary) + } + fmt.Fprintf(&b, " • [%s] %s\n", o.SourceID, truncateStr(line, 160)) + } + b.WriteString("\n") + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_thread_link ─────────────────────────────────────── + threadLinkTool := mcp.NewTool("knox_thread_link", + mcp.WithDescription("Link an observation to a thread by observation ID. Use knox_get first to find observation IDs."), + mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")), + mcp.WithNumber("observation_id", mcp.Required(), mcp.Description("Observation ID to link")), + mcp.WithString("relevance", mcp.Description("Why this observation matters to the thread")), + ) + + s.AddTool(threadLinkTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + threadID := int64(clampInt(req, "thread_id", 0, 1, 999999)) + obsID := int64(clampInt(req, "observation_id", 0, 1, 999999)) + relevance := getString(req, "relevance", "") + if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil { + return errorResult(err.Error()), nil + } + return mcp.NewToolResultText(fmt.Sprintf("Linked observation #%d to thread #%d", obsID, threadID)), nil + }) + + // ─── knox_thread_search ───────────────────────────────────── + threadSearchTool := mcp.NewTool("knox_thread_search", + mcp.WithDescription("Search threads by motivation, title, or tags. Find in-flight knowledge work by intent."), + mcp.WithString("query", mcp.Required(), mcp.Description("Search motivation, title, or tags")), + ) + + s.AddTool(threadSearchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + query := getString(req, "query", "") + threads, err := kdb.SearchThreadsByMotivation(query) + if err != nil { + return errorResult(err.Error()), nil + } + if len(threads) == 0 { + return mcp.NewToolResultText(fmt.Sprintf("No threads matching %q", query)), nil + } + var b strings.Builder + for _, t := range threads { + b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount)) + if t.Motivation != "" { + b.WriteString(fmt.Sprintf(" %s\n", t.Motivation)) + } + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_thread_link_entry ───────────────────────────────── + threadLinkEntryTool := mcp.NewTool("knox_thread_link_entry", + mcp.WithDescription("Link an entry to a thread via provenance graph. Use after knox_search to connect findings to intent."), + mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")), + mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint")), + mcp.WithString("relation", mcp.Description("Relation type: produced (default), references, motivated")), + ) + + s.AddTool(threadLinkEntryTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + threadID := int64(clampInt(req, "thread_id", 0, 1, 999999)) + fp := getString(req, "fingerprint", "") + relation := getString(req, "relation", "produced") + if err := kdb.LinkEntryToThread(threadID, fp, relation); err != nil { + return errorResult(err.Error()), nil + } + return mcp.NewToolResultText(fmt.Sprintf("Linked %s to thread #%d [%s]", shortFP(fp), threadID, relation)), nil + }) + + // ─── knox_thread_relate ───────────────────────────────────── + threadRelateTool := mcp.NewTool("knox_thread_relate", + mcp.WithDescription("Relate two threads (parent spawns child, threads merge, etc.). Builds the thread-inquiry graph."), + mcp.WithNumber("parent_id", mcp.Required(), mcp.Description("Parent thread ID")), + mcp.WithNumber("child_id", mcp.Required(), mcp.Description("Child thread ID")), + mcp.WithString("relation", mcp.Description("Relation: spawned (default), merged_into, references, supersedes")), + ) + + s.AddTool(threadRelateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + parentID := int64(clampInt(req, "parent_id", 0, 1, 999999)) + childID := int64(clampInt(req, "child_id", 0, 1, 999999)) + relation := getString(req, "relation", "spawned") + if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil { + return errorResult(err.Error()), nil + } + return mcp.NewToolResultText(fmt.Sprintf("Related thread #%d -> #%d [%s]", parentID, childID, relation)), nil + }) + + // ─── knox_thread_golden ───────────────────────────────────── + goldenTool := mcp.NewTool("knox_thread_golden", + mcp.WithDescription("Set, get, or clear the golden thread. The golden thread represents your current focus — new observations auto-link to it."), + mcp.WithNumber("thread_id", mcp.Description("Thread ID to set as golden. Omit to query current. Pass 0 to clear.")), + ) + + s.AddTool(goldenTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + threadID := int64(clampInt(req, "thread_id", 0, 0, 999999)) + + if threadID == 0 { + // Query or clear + current, _ := kdb.GoldenThreadID() + if current == 0 { + return mcp.NewToolResultText("No golden thread set."), nil + } + // If thread_id was explicitly 0 and a golden exists, clear it + if _, ok := req.Params.Arguments["thread_id"]; ok { + kdb.SetGoldenThread(0) + t, _ := kdb.GetThread(current) + return mcp.NewToolResultText(fmt.Sprintf("Golden thread cleared (was #%d: %s)", current, t.Title)), nil + } + t, _ := kdb.GetThread(current) + return mcp.NewToolResultText(fmt.Sprintf("Golden thread: #%d %s [%s]\n %s", t.ID, t.Title, t.Status, t.Motivation)), nil + } + + if err := kdb.SetGoldenThread(threadID); err != nil { + return errorResult(err.Error()), nil + } + t, _ := kdb.GetThread(threadID) + return mcp.NewToolResultText(fmt.Sprintf("Golden thread set to #%d: %s", threadID, t.Title)), nil + }) + + // ─── knox_topics ─────────────────────────────────────────── + topicsTool := mcp.NewTool("knox_topics", + mcp.WithDescription("List auto-detected topic clusters from TF-IDF analysis. Shows what topics appear across all sources."), + mcp.WithNumber("limit", mcp.Description("Max topics (default 10)")), + ) + + s.AddTool(topicsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + limit := clampInt(req, "limit", 10, 1, 30) + entries, _ := kdb.RecentEntries(2000) + tfidf := index.BuildTFIDF(entries) + clusters := tfidf.Cluster(2, limit) + + if len(clusters) == 0 { + return mcp.NewToolResultText("No topic clusters detected."), nil + } + var b strings.Builder + for i, c := range clusters { + total := 0 + for _, v := range c.BySource { + total += v + } + b.WriteString(fmt.Sprintf("%d. %s (%d entries)\n", i+1, c.Name, total)) + var sources []string + for src, count := range c.BySource { + sources = append(sources, fmt.Sprintf("%s:%d", src, count)) + } + b.WriteString(fmt.Sprintf(" Sources: %s\n", strings.Join(sources, ", "))) + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_gaps ───────────────────────────────────────────── + gapsTool := mcp.NewTool("knox_gaps", + mcp.WithDescription("Find topics with browser activity but no skill coverage. Identifies candidates for new skill creation."), + ) + + s.AddTool(gapsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + entries, _ := kdb.RecentEntries(2000) + + skillKeywords := make(map[string]bool) + for _, e := range entries { + if e.SourceID == "skills-catalog" { + for _, w := range index.Tokenize(e.Title + " " + e.Summary) { + skillKeywords[w] = true + } + } + } + + tfidf := index.BuildTFIDF(entries) + clusters := tfidf.Cluster(2, 15) + gaps := tfidf.Gaps(clusters, skillKeywords) + + if len(gaps) == 0 { + return mcp.NewToolResultText("No gaps found."), nil + } + var b strings.Builder + b.WriteString(fmt.Sprintf("Found %d topics with browser activity but NO skill coverage:\n\n", len(gaps))) + for i, g := range gaps { + browserCount := g.BySource["browser-history"] + b.WriteString(fmt.Sprintf("%d. %s — %d browser visits\n", i+1, g.Name, browserCount)) + } + return mcp.NewToolResultText(b.String()), nil + }) + + // ─── knox_git_status ─────────────────────────────────────── + gitStatusTool := mcp.NewTool("knox_git_status", + mcp.WithDescription("List local git repository status captured by Knox (branch, clean/dirty, ahead/behind, HEAD, last commit). Pass project to filter."), + mcp.WithString("project", mcp.Description("Optional project/repo name substring filter")), + ) + + s.AddTool(gitStatusTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + entries, err := kdb.EntriesBySource("git") + if err != nil { + return errorResult(err.Error()), nil + } + filter, _ := req.Params.Arguments["project"].(string) + + if len(entries) == 0 { + return mcp.NewToolResultText("No git status entries yet. Run `knox git` or wait for the daemon's git timer."), nil + } + + var b strings.Builder + shown := 0 + for _, e := range entries { + if filter != "" && !strings.Contains(strings.ToLower(e.Project), strings.ToLower(filter)) { + continue + } + b.WriteString(fmt.Sprintf("%-24s %s\n", e.Project, e.Summary)) + shown++ + } + b.WriteString(fmt.Sprintf("\n%d / %d repos", shown, len(entries))) + return mcp.NewToolResultText(b.String()), nil + }) + + return s +} + +// ─── Format helpers ────────────────────────────────────────── + +func formatEntries(entries []db.Entry, detail detailLevel, kdb *db.KnoxDB) string { + switch detail { + case detailBrief: + return formatEntriesBrief(entries) + case detailFull: + var b strings.Builder + for i, e := range entries { + if i > 0 { + b.WriteString("---\n") + } + b.WriteString(formatEntryFull(&e)) + } + return b.String() + default: + var b strings.Builder + for _, e := range entries { + b.WriteString(formatEntryNormal(&e)) + b.WriteString("\n") + } + return b.String() + } +} + +func formatEntriesBrief(entries []db.Entry) string { + var b strings.Builder + for _, e := range entries { + b.WriteString(fmt.Sprintf("%-8s | %-40s | %-14s | %s\n", + shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project)) + } + return b.String() +} + +func formatEntryNormal(e *db.Entry) string { + return fmt.Sprintf("%-8s %-40s [%-14s %-4s] seen:%d conf:%.1f\n %s", + shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project, + e.RefCount, e.LastConfidence, truncateStr(e.Summary, 90)) +} + +func formatEntryFull(e *db.Entry) string { + return fmt.Sprintf(`fingerprint: %s +title: %s +source: %s +path: %s +project: %s +type: %s +first_seen: %s +last_seen: %s +ref_count: %d +confidence: %.1f +summary: %s`, + e.Fingerprint, e.Title, e.SourceID, e.SourcePath, e.Project, e.ContentType, + e.FirstSeen, e.LastSeen, e.RefCount, e.LastConfidence, e.Summary) +} + +// ─── Scope filtering ───────────────────────────────────────── + +func filterByScope(entries []db.Entry, scope string) []db.Entry { + switch strings.ToLower(scope) { + case "skills": + return filterSource(entries, "skills-catalog") + case "sessions": + return filterSource(entries, "opencode-session") + case "logs": + return filterSource(entries, "opencode-log") + default: + return entries + } +} + +func filterSource(entries []db.Entry, sourceID string) []db.Entry { + var filtered []db.Entry + for _, e := range entries { + if e.SourceID == sourceID { + filtered = append(filtered, e) + } + } + return filtered +} + +// ─── Argument helpers ──────────────────────────────────────── + +func getString(req mcp.CallToolRequest, key, def string) string { + if v, ok := req.Params.Arguments[key].(string); ok { + return v + } + return def +} + +func clampInt(req mcp.CallToolRequest, key string, def, min, max int) int { + if v, ok := req.Params.Arguments[key].(float64); ok { + n := int(v) + if n < min { + return min + } + if n > max { + return max + } + return n + } + return def +} + +func shortFP(fp string) string { + if len(fp) > 8 { + return fp[:8] + } + return fp +} + +func truncateStr(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) + "..." +} + +func errorResult(msg string) *mcp.CallToolResult { + return &mcp.CallToolResult{ + IsError: true, + Content: []mcp.Content{mcp.NewTextContent(msg)}, + } +} diff --git a/internal/cmd/obsidian.go b/internal/cmd/obsidian.go new file mode 100644 index 0000000..26b8e88 --- /dev/null +++ b/internal/cmd/obsidian.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" +) + +func NewObsidianCmd(kdb *db.KnoxDB) *cobra.Command { + var vault string + cmd := &cobra.Command{ + Use: "obsidian", + Short: "Ingest Obsidian notes into the knowledge index", + Long: `Scans an Obsidian vault for markdown notes, extracts frontmatter +(title, tags, dates), and fingerprints them into the index. + +Auto-detects vault path from ~/.config/obsidian/obsidian.json +or override with --vault.`, + RunE: func(c *cobra.Command, args []string) error { + ing := ingest.NewObsidianIngester(vault) + log.Printf("[knox] ingesting Obsidian notes...") + + results, err := ing.IngestAll() + if err != nil { + return fmt.Errorf("obsidian ingest: %w", err) + } + + if len(results) == 0 { + fmt.Println("No Obsidian notes found.") + return nil + } + + var newCount, dupCount int + for _, result := range results { + _, isNew, err := kdb.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + LineStart: result.LineStart, + LineEnd: result.LineEnd, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "obsidian_ingest", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + continue + } + if isNew { + newCount++ + } else { + dupCount++ + } + } + + fmt.Printf("Obsidian notes: %d new, %d updated\n", newCount, dupCount) + return nil + }, + } + cmd.Flags().StringVarP(&vault, "vault", "v", "", "Obsidian vault path (auto-detect if empty)") + return cmd +} diff --git a/internal/cmd/status.go b/internal/cmd/status.go new file mode 100644 index 0000000..2c0cbd6 --- /dev/null +++ b/internal/cmd/status.go @@ -0,0 +1,161 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/index" +) + +func paginate(entries []db.Entry, page, limit int) []db.Entry { + start := (page - 1) * limit + if start >= len(entries) { + return nil + } + end := start + limit + if end > len(entries) { + end = len(entries) + } + return entries[start:end] +} + +func NewStatusCmd(kdb *db.KnoxDB) *cobra.Command { + var jsonOut bool + cmd := &cobra.Command{ + Use: "status", + Short: "Show knowledge index statistics", + RunE: func(c *cobra.Command, args []string) error { + stats, err := kdb.Stats() + if err != nil { + return err + } + if jsonOut { + b, _ := json.MarshalIndent(stats, "", " ") + fmt.Println(string(b)) + return nil + } + fmt.Println("Knox Knowledge Index") + fmt.Println("====================") + for k, v := range stats { + switch val := v.(type) { + case int: + fmt.Printf(" %-24s %d\n", k+":", val) + case string: + fmt.Printf(" %-24s %s\n", k+":", val) + default: + fmt.Printf(" %-24s %v\n", k+":", val) + } + } + return nil + }, + } + cmd.Flags().BoolVar(&jsonOut, "json", false, "Output as JSON") + return cmd +} + +func NewQueryCmd(kdb *db.KnoxDB) *cobra.Command { + var page, limit int + cmd := &cobra.Command{ + Use: "query [search text]", + Short: "Search the knowledge index", + Long: "Search the knowledge index. Reads from stdin pipe if no positional arg given.", + RunE: func(c *cobra.Command, args []string) error { + query := "" + if len(args) > 0 { + query = args[0] + } else { + stdin, _ := os.ReadFile(os.Stdin.Name()) + query = strings.TrimSpace(string(stdin)) + } + if query == "" { + return fmt.Errorf("search query required (positional arg or stdin pipe)") + } + results, err := kdb.Search(query, limit) + if err != nil { + return err + } + results = paginate(results, page, limit) + if len(results) == 0 { + fmt.Println("No results found.") + return nil + } + fmt.Printf("Found %d results for %q (page %d, %d per page):\n\n", len(results), query, page, limit) + for _, r := range results { + fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(r.Fingerprint), truncateStr(r.Title, 30), r.Project, r.SourceID) + if r.Summary != "" { + fmt.Printf(" %-8s %s\n", "", truncateStr(r.Summary, 80)) + } + fmt.Println() + } + return nil + }, + } + cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)") + cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page") + return cmd +} + +func NewRecentCmd(kdb *db.KnoxDB) *cobra.Command { + var page, limit int + cmd := &cobra.Command{ + Use: "recent", + Short: "Show recent knowledge entries", + RunE: func(c *cobra.Command, args []string) error { + all, err := kdb.RecentEntries(limit * 10) + if err != nil { + return err + } + entries := paginate(all, page, limit) + if len(entries) == 0 { + fmt.Println("No entries yet. Run `knox watch` to start ingesting.") + return nil + } + fmt.Printf("Recent %d entries (page %d, %d per page):\n\n", len(entries), page, limit) + for _, e := range entries { + fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(e.Fingerprint), truncateStr(e.Title, 30), e.Project, e.SourceID) + } + return nil + }, + } + cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)") + cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page") + return cmd +} + +func NewReflectCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "reflect", + Short: "Process pending sessions for reflection", + Long: "Scans for sessions not yet indexed, runs reflection, and reports findings.", + RunE: func(c *cobra.Command, args []string) error { + reflector := index.NewReflector(kdb) + results, err := reflector.Reflect() + if err != nil { + return err + } + if len(results) == 0 { + fmt.Println("All caught up — no pending sessions to reflect on.") + return nil + } + fmt.Printf("Reflected %d sessions:\n\n", len(results)) + fmt.Print(reflector.FormatResult(results)) + + gaps, _ := reflector.Gaps() + if len(gaps) > 0 { + fmt.Printf("\nProjects with indexed knowledge:\n") + for _, g := range gaps { + fmt.Printf(" %-20s %d entries\n", g.Project, g.TotalEntries) + } + } + return nil + }, + } +} + +func getHomeDir() (string, error) { + return os.UserHomeDir() +} diff --git a/internal/cmd/thread.go b/internal/cmd/thread.go new file mode 100644 index 0000000..2bdd293 --- /dev/null +++ b/internal/cmd/thread.go @@ -0,0 +1,471 @@ +package cmd + +import ( + "fmt" + "strconv" + "strings" + + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/watch" + "github.com/spf13/cobra" +) + +func NewThreadCmd(kdb *db.KnoxDB) *cobra.Command { + cmd := &cobra.Command{ + Use: "thread", + Short: "Manage knowledge development threads (motivation/intent tracking)", + Long: `Threads capture WHY observations were collected — the intent, +motivation, and in-flight knowledge development work. + +Subcommands: create, list, link, link-entry, relate, golden, prune, rescan, close, show, note`, + } + cmd.AddCommand(newThreadCreateCmd(kdb)) + cmd.AddCommand(newThreadAutoCmd(kdb)) + cmd.AddCommand(newThreadDraftCmd(kdb)) + cmd.AddCommand(newThreadUpdateCmd(kdb)) + cmd.AddCommand(newThreadListCmd(kdb)) + cmd.AddCommand(newThreadLinkCmd(kdb)) + cmd.AddCommand(newThreadCloseCmd(kdb)) + cmd.AddCommand(newThreadShowCmd(kdb)) + cmd.AddCommand(newThreadNoteCmd(kdb)) + cmd.AddCommand(newThreadLinkEntryCmd(kdb)) + cmd.AddCommand(newThreadRelateCmd(kdb)) + cmd.AddCommand(newThreadGoldenCmd(kdb)) + cmd.AddCommand(newThreadPruneCmd(kdb)) + cmd.AddCommand(newThreadRescanCmd(kdb)) + return cmd +} + +func newThreadCreateCmd(kdb *db.KnoxDB) *cobra.Command { + var motivation, priority, tags, provenance string + cmd := &cobra.Command{ + Use: "create ", + Short: "Create a new knowledge development thread", + Args: cobra.MinimumNArgs(1), + RunE: func(c *cobra.Command, args []string) error { + title := strings.Join(args, " ") + id, err := kdb.CreateThread(title, motivation, priority, tags, provenance) + if err != nil { + return err + } + fmt.Printf("Created thread #%d: %s\n", id, title) + return nil + }, + } + cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "Why this thread exists (the 'so what')") + cmd.Flags().StringVarP(&priority, "priority", "p", "medium", "Priority: high, medium, low") + cmd.Flags().StringVarP(&tags, "tags", "t", "", "Comma-separated tags") + cmd.Flags().StringVarP(&provenance, "provenance", "", "{}", "JSON provenance metadata") + return cmd +} + +func newThreadAutoCmd(kdb *db.KnoxDB) *cobra.Command { + var dryRun bool + cmd := &cobra.Command{ + Use: "auto", + Short: "Auto-create threads from high-intent topic clusters (heuristic, no LLM)", + Long: `Clusters recent observations with TF-IDF and promotes clusters that cross +the "intent bar" into threads, auto-linking their observations. + +The intent bar (tunable via KNOX_THREAD_* env vars) requires: enough distinct +sources, at least one non-passive work source (git/gitea/session/obsidian), and +observations observed recently. Use --dry-run to preview without writing.`, + RunE: func(c *cobra.Command, args []string) error { + threader := watch.NewAutoThreader(kdb) + threader.DryRun = dryRun + created, linked, err := threader.AutoThread() + if err != nil { + return err + } + fmt.Printf("\nAuto-thread: %d created, %d observations linked\n", created, linked) + return nil + }, + } + cmd.Flags().BoolVar(&dryRun, "dry-run", dryRun, "Preview candidate threads without writing") + return cmd +} + +func newThreadDraftCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "draft", + Short: "List auto-threaded candidates for LLM title/motivation polish", + Long: `Shows threads created by the heuristic auto-threader ('auto_thread' +provenance) alongside their linked observations, so an LLM or editor can write +a better title and motivation. Apply changes with ` + "`knox thread update <id>`" + `.`, + RunE: func(c *cobra.Command, args []string) error { + threads, err := kdb.ListThreads("") + if err != nil { + return err + } + var shown int + for _, t := range threads { + if !isAutoThreaded(t) { + continue + } + obs, _ := kdb.ThreadObservations(t.ID) + fmt.Printf("Thread #%d [%s] %d obs\n", t.ID, t.Priority, t.EntryCount) + fmt.Printf(" title: %s\n", t.Title) + fmt.Printf(" motivation: %s\n", t.Motivation) + if t.Tags != "[]" { + fmt.Printf(" tags: %s\n", t.Tags) + } + fmt.Println(" linked observations:") + for _, o := range obs { + title := truncateStr(o.Title, 70) + if o.Summary != "" { + title += " — " + truncateStr(o.Summary, 60) + } + fmt.Printf(" • [%s] %s\n", o.SourceID, title) + } + fmt.Println() + shown++ + } + if shown == 0 { + fmt.Println("No auto-threaded candidates found. Run `knox thread auto` (or wait for the watch tick) first.") + } + return nil + }, + } +} + +func isAutoThreaded(t db.Thread) bool { + if strings.Contains(t.Provenance, "auto_thread") { + return true + } + return strings.HasPrefix(t.Motivation, "Auto-detected") +} + +func newThreadUpdateCmd(kdb *db.KnoxDB) *cobra.Command { + var title, motivation, priority, tags string + cmd := &cobra.Command{ + Use: "update <thread-id>", + Short: "Rewrite a thread's title/motivation/priority/tags", + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + changed, err := kdb.UpdateThread(id, title, motivation, priority, tags) + if err != nil { + return err + } + if !changed { + fmt.Println("No changes applied.") + return nil + } + fmt.Printf("Updated thread #%d\n", id) + return nil + }, + } + cmd.Flags().StringVarP(&title, "title", "t", "", "New title") + cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "New motivation") + cmd.Flags().StringVarP(&priority, "priority", "p", "", "New priority (high/medium/low)") + cmd.Flags().StringVar(&tags, "tags", "", "New space-separated tags") + return cmd +} + +func newThreadListCmd(kdb *db.KnoxDB) *cobra.Command { + var status string + cmd := &cobra.Command{ + Use: "list", + Short: "List knowledge development threads", + RunE: func(c *cobra.Command, args []string) error { + threads, err := kdb.ListThreads(status) + if err != nil { + return err + } + if len(threads) == 0 { + fmt.Println("No threads. Create one with `knox thread create`.") + return nil + } + fmt.Printf("%-4s %-40s %-10s %-8s %s\n", "ID", "Title", "Status", "Priority", "Entries") + fmt.Println(strings.Repeat("-", 75)) + for _, t := range threads { + fmt.Printf("%-4d %-40s %-10s %-8s %d\n", t.ID, truncateStr(t.Title, 38), t.Status, t.Priority, t.EntryCount) + if t.Motivation != "" { + fmt.Printf(" %s\n", truncateStr(t.Motivation, 70)) + } + } + return nil + }, + } + cmd.Flags().StringVarP(&status, "status", "s", "", "Filter by status (active, stalled, resolved)") + return cmd +} + +func newThreadLinkCmd(kdb *db.KnoxDB) *cobra.Command { + var relevance string + cmd := &cobra.Command{ + Use: "link <thread-id> <observation-id>", + Short: "Link an observation to a thread", + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + threadID, _ := strconv.ParseInt(args[0], 10, 64) + obsID, _ := strconv.ParseInt(args[1], 10, 64) + if threadID == 0 || obsID == 0 { + return fmt.Errorf("thread-id and observation-id must be numeric") + } + if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil { + return err + } + fmt.Printf("Linked observation #%d to thread #%d\n", obsID, threadID) + return nil + }, + } + cmd.Flags().StringVarP(&relevance, "relevance", "r", "", "Why this observation matters to the thread") + return cmd +} + +func newThreadCloseCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "close <thread-id>", + Short: "Mark a thread as resolved", + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + thread, _ := kdb.GetThread(id) + if thread == nil { + return fmt.Errorf("thread #%d not found", id) + } + if err := kdb.CloseThread(id); err != nil { + return err + } + fmt.Printf("Closed thread #%d: %s\n", id, thread.Title) + return nil + }, + } +} + +func newThreadShowCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "show <thread-id>", + Short: "Show thread details with provenance, linked observations, and notes", + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + t, err := kdb.GetThread(id) + if err != nil { + return err + } + if t == nil { + return fmt.Errorf("thread #%d not found", id) + } + goldenID, _ := kdb.GoldenThreadID() + fmt.Printf("Thread #%d: %s", t.ID, t.Title) + if goldenID == id { + fmt.Print(" [GOLDEN]") + } + fmt.Println() + fmt.Printf(" Status: %s\n", t.Status) + fmt.Printf(" Priority: %s\n", t.Priority) + fmt.Printf(" Created: %s\n", t.CreatedAt) + fmt.Printf(" Updated: %s\n", t.UpdatedAt) + if t.ResolvedAt != "" { + fmt.Printf(" Resolved: %s\n", t.ResolvedAt) + } + if t.Motivation != "" { + fmt.Printf(" Motivation: %s\n", t.Motivation) + } + if t.Tags != "[]" { + fmt.Printf(" Tags: %s\n", t.Tags) + } + if t.Provenance != "{}" { + fmt.Printf(" Provenance: %s\n", t.Provenance) + } + obs, _ := kdb.ThreadObservations(id) + if len(obs) > 0 { + fmt.Printf("\nLinked observations (%d):\n", len(obs)) + for _, o := range obs { + fmt.Printf(" #%-4d %-30s [%s] trigger=%s\n", o.ID, truncateStr(o.Title, 28), o.SourceID, o.Trigger) + } + } + provenance, _ := kdb.ThreadProvenance(id) + if len(provenance) > 0 { + fmt.Printf("\nProvenance graph (%d links):\n", len(provenance)) + for _, p := range provenance { + fmt.Printf(" %s --[%s]--> thread:%d\n", shortFP(p.Fingerprint), p.Relation, id) + } + } + notes, _ := kdb.ThreadNotes(id) + if len(notes) > 0 { + fmt.Printf("\nNotes (%d):\n", len(notes)) + for _, n := range notes { + fmt.Printf(" [%s] %s\n", n.CreatedAt, n.Note) + } + } + return nil + }, + } +} + +func newThreadLinkEntryCmd(kdb *db.KnoxDB) *cobra.Command { + var relation string + cmd := &cobra.Command{ + Use: "link-entry <thread-id> <entry-fingerprint>", + Short: "Link an entry to a thread via provenance graph", + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + if err := kdb.LinkEntryToThread(id, args[1], relation); err != nil { + return err + } + fmt.Printf("Linked entry %s to thread #%d [%s]\n", shortFP(args[1]), id, relation) + return nil + }, + } + cmd.Flags().StringVarP(&relation, "relation", "r", "produced", "Relation type (produced, references, motivated)") + return cmd +} + +func newThreadRelateCmd(kdb *db.KnoxDB) *cobra.Command { + var relation string + cmd := &cobra.Command{ + Use: "relate <parent-id> <child-id>", + Short: "Relate two threads (parent spawned child, etc.)", + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + parentID, _ := strconv.ParseInt(args[0], 10, 64) + childID, _ := strconv.ParseInt(args[1], 10, 64) + if parentID == 0 || childID == 0 { + return fmt.Errorf("both thread IDs must be numeric") + } + if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil { + return err + } + fmt.Printf("Related thread #%d -> #%d [%s]\n", parentID, childID, relation) + return nil + }, + } + cmd.Flags().StringVarP(&relation, "relation", "r", "spawned", "Relation type (spawned, merged_into, references, supersedes)") + return cmd +} + +func newThreadGoldenCmd(kdb *db.KnoxDB) *cobra.Command { + var unset bool + cmd := &cobra.Command{ + Use: "golden [thread-id]", + Short: "Set/show/clear the golden thread (current focus). Auto-links new observations.", + RunE: func(c *cobra.Command, args []string) error { + if unset { + kdb.SetGoldenThread(0) + fmt.Println("Golden thread cleared.") + return nil + } + if len(args) > 0 { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + if err := kdb.SetGoldenThread(id); err != nil { + return err + } + t, _ := kdb.GetThread(id) + if t != nil { + fmt.Printf("Golden thread set to #%d: %s\n", id, t.Title) + } + return nil + } + goldenID, _ := kdb.GoldenThreadID() + if goldenID == 0 { + fmt.Println("No golden thread set.") + return nil + } + t, _ := kdb.GetThread(goldenID) + if t != nil { + fmt.Printf("Golden thread: #%d %s [%s/%s]\n %s\n", t.ID, t.Title, t.Status, t.Priority, t.Motivation) + } + return nil + }, + } + cmd.Flags().BoolVar(&unset, "unset", false, "Clear the golden thread") + return cmd +} + +func newThreadNoteCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "note <thread-id> <text>", + Short: "Add a note to a thread", + Args: cobra.MinimumNArgs(2), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + note := strings.Join(args[1:], " ") + nid, err := kdb.AddThreadNote(id, note) + if err != nil { + return err + } + fmt.Printf("Added note #%d to thread #%d\n", nid, id) + return nil + }, + } +} + +func newThreadPruneCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "prune <thread-id>", + Short: "Remove irrelevant auto-linked observations from a thread", + Long: `Scans auto-linked observations and removes ones that don't share keywords with the thread's title, motivation, or tags.`, + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + t, err := kdb.GetThread(id) + if err != nil || t == nil { + return fmt.Errorf("thread #%d not found", id) + } + keywords, bigrams, err := kdb.ThreadRelevanceProfile(id) + if err != nil { + return err + } + if len(keywords) == 0 { + return fmt.Errorf("no significant keywords found in thread") + } + count, err := kdb.PruneThread(id) + if err != nil { + return err + } + fmt.Printf("Pruned %d irrelevant links from thread #%d (%s)\n", count, id, t.Title) + fmt.Printf(" Kept observations matching: %v (bigrams: %v)\n", keywords, bigrams) + return nil + }, + } +} + +func newThreadRescanCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "rescan <thread-id>", + Short: "Link unlinked observations that pass the thread's relevance profile", + Long: `Scans all observations and links ones matching the thread's DF-filtered keyword profile. Inverse of prune.`, + Args: cobra.ExactArgs(1), + RunE: func(c *cobra.Command, args []string) error { + id, _ := strconv.ParseInt(args[0], 10, 64) + if id == 0 { + return fmt.Errorf("thread-id must be numeric") + } + t, err := kdb.GetThread(id) + if err != nil || t == nil { + return fmt.Errorf("thread #%d not found", id) + } + count, err := kdb.RescanThread(id) + if err != nil { + return err + } + fmt.Printf("Linked %d observations to thread #%d (%s)\n", count, id, t.Title) + return nil + }, + } +} diff --git a/internal/cmd/topics.go b/internal/cmd/topics.go new file mode 100644 index 0000000..32a0c77 --- /dev/null +++ b/internal/cmd/topics.go @@ -0,0 +1,135 @@ +package cmd + +import ( + "fmt" + "regexp" + "strings" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/index" +) + +func NewTopicsCmd(kdb *db.KnoxDB) *cobra.Command { + cmd := &cobra.Command{ + Use: "topics", + Short: "Auto-detect topic clusters from TF-IDF analysis of the knowledge index", + Long: `Scans all entries, computes TF-IDF term scores, and clusters related +observations into topics. Shows gap analysis for browser-researched +topics not covered by skills. + +Subcommands: list, gaps`, + } + + cmd.AddCommand(newTopicListCmd(kdb)) + cmd.AddCommand(newTopicGapsCmd(kdb)) + + return cmd +} + +func newTopicListCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List auto-detected topic clusters", + RunE: func(c *cobra.Command, args []string) error { + entries, err := kdb.RecentEntries(2000) + if err != nil { + return err + } + + tfidf := index.BuildTFIDF(entries) + clusters := tfidf.Cluster(2, 15) + + if len(clusters) == 0 { + fmt.Println("No topic clusters detected.") + return nil + } + + fmt.Printf("%-3s %-50s %-8s %s\n", "ID", "Topic", "Entries", "Sources") + fmt.Println(strings.Repeat("-", 75)) + for i, c := range clusters { + total := 0 + for _, v := range c.BySource { + total += v + } + sources := "" + for src, count := range c.BySource { + if sources != "" { + sources += ", " + } + sources += fmt.Sprintf("%s:%d", shortSource(src), count) + } + fmt.Printf("%-3d %-50s %-8d %s\n", i+1, truncateStr(c.Name, 48), total, sources) + } + return nil + }, + } +} + +func newTopicGapsCmd(kdb *db.KnoxDB) *cobra.Command { + return &cobra.Command{ + Use: "gaps", + Short: "Show topics with browser activity but no skill coverage", + Long: `Cross-references browser history against the skills catalog. +Flags topics that have significant browser research but zero skill entries. + +These are candidates for new skill creation.`, + RunE: func(c *cobra.Command, args []string) error { + entries, err := kdb.RecentEntries(2000) + if err != nil { + return err + } + + // Build skill keyword set + skillKeywords := make(map[string]bool) + for _, e := range entries { + if e.SourceID == "skills-catalog" { + for _, w := range tokenize(e.Title + " " + e.Summary) { + skillKeywords[w] = true + } + } + } + + tfidf := index.BuildTFIDF(entries) + clusters := tfidf.Cluster(2, 15) + gaps := tfidf.Gaps(clusters, skillKeywords) + + if len(gaps) == 0 { + fmt.Println("No gaps found — all browser-researched topics have skill coverage.") + return nil + } + + fmt.Printf("\nFound %d topics with browser activity but NO skill coverage:\n\n", len(gaps)) + for i, g := range gaps { + browserCount := g.BySource["browser-history"] + fmt.Printf("%d. %s\n", i+1, g.Name) + fmt.Printf(" %d browser visits · keywords: %v\n", browserCount, g.Keywords) + fmt.Println() + } + + fmt.Println("Consider creating skills for these topics with `knox asses` or via opencode.") + return nil + }, + } +} + +func shortSource(s string) string { + switch s { + case "opencode-session": + return "session" + case "skills-catalog": + return "skill" + case "browser-history": + return "browser" + case "obsidian": + return "notes" + case "opencode-log": + return "logs" + } + return s +} + +func tokenize(text string) []string { + re := regexp.MustCompile(`[a-z]{4,}`) + return re.FindAllString(strings.ToLower(text), -1) +} diff --git a/internal/cmd/watch.go b/internal/cmd/watch.go new file mode 100644 index 0000000..bc995c8 --- /dev/null +++ b/internal/cmd/watch.go @@ -0,0 +1,44 @@ +package cmd + +import ( + "io" + "log" + + "github.com/spf13/cobra" + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/watch" +) + +func NewWatchCmd(kdb *db.KnoxDB) *cobra.Command { + var dirs []string + var quiet bool + cmd := &cobra.Command{ + Use: "watch", + Short: "Watch directories for new session data and ingest into index", + Run: func(cmd *cobra.Command, args []string) { + if len(dirs) == 0 { + dirs = defaultWatchDirs() + } + if quiet { + log.SetOutput(io.Discard) + } + log.Printf("[knox] starting watcher, scanning %d directories", len(dirs)) + w := watch.New(kdb, dirs) + if err := w.Start(); err != nil { + log.Fatalf("watcher error: %v", err) + } + }, + } + cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to watch (default: opencode storage/log + skills)") + cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Suppress per-entry log output") + return cmd +} + +func defaultWatchDirs() []string { + home, _ := getHomeDir() + return []string{ + home + "/.local/share/opencode/storage/session_diff", + home + "/.local/share/opencode/log", + home + "/.skills", + } +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 0000000..f9211b9 --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,1325 @@ +package db + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + _ "modernc.org/sqlite" +) + +const goldenMinConfidence = 0.7 + +type KnoxDB struct { + db *sql.DB +} + +type Observation struct { + ID int64 + Fingerprint string + SourceID string + SourcePath string + Project string + ContentType string + Title string + Summary string + CollectedAt string + CreatedAt string + LineStart int + LineEnd int + Confidence float64 + IngesterVersion string + Trigger string + Count int // duplicates collapsed by ObservationsByFingerprint +} + +type Entry struct { + Fingerprint string + SourceID string + SourcePath string + Project string + ContentType string + Title string + Summary string + FirstSeen string + LastSeen string + CreatedAt string + RefCount int + LastConfidence float64 +} + +type SessionRow struct { + SessionID string + Project string + Title string + Status string + AgentCount int + StartedAt string + EndedAt string + LastSeen string + Indexed bool +} + +type ObservationRecord struct { + Fingerprint string + SourceID string + SourcePath string + Project string + ContentType string + Title string + Summary string + CreatedAt string + LineStart int + LineEnd int + Confidence float64 + IngesterVersion string + Trigger string + Provenance string +} + +func Open(path string) (*KnoxDB, error) { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("create db dir: %w", err) + } + + db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, fmt.Errorf("open db: %w", err) + } + + if _, err := db.Exec(Schema); err != nil { + return nil, fmt.Errorf("init schema: %w", err) + } + + // Migrations: safe ALTER TABLE for existing databases + migrations := []string{ + "ALTER TABLE threads ADD COLUMN provenance TEXT DEFAULT '{}'", + "ALTER TABLE entries ADD COLUMN created_at TEXT DEFAULT ''", + } + for _, m := range migrations { + if _, err := db.Exec(m); err != nil && !strings.Contains(err.Error(), "duplicate column") { + return nil, fmt.Errorf("migration failed: %w", err) + } + } + + return &KnoxDB{db: db}, nil +} + +func (k *KnoxDB) Close() error { return k.db.Close() } + +// RecordObservation appends an immutable observation and updates the entries cache. +// Idempotent: if the latest observation for this fingerprint has an identical +// content signature, nothing is recorded — re-ingesting unchanged content is a no-op. +func (k *KnoxDB) RecordObservation(o ObservationRecord) (obsID int64, isNew bool, err error) { + tx, err := k.db.Begin() + if err != nil { + return 0, false, fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + now := time.Now().UTC().Format(time.RFC3339) + + // Dedup: compare against the latest observation of the same fingerprint. + // For content sources (LineEnd > 0) the extent/summary IS the change signal — + // created_at is just the file mtime, which can twitch without content change + // (live log files). For extent-less sources (browser, gitea), created_at is + // the only change signal (revisit, issue update) so it must match. + var prevID int64 + var pTitle, pSummary, pProject, pCreated string + var pStart, pEnd int + err = tx.QueryRow( + `SELECT id, COALESCE(title,''), COALESCE(summary,''), COALESCE(project,''), + COALESCE(created_at,''), COALESCE(line_start,0), COALESCE(line_end,0) + FROM observations WHERE fingerprint=? ORDER BY id DESC LIMIT 1`, o.Fingerprint, + ).Scan(&prevID, &pTitle, &pSummary, &pProject, &pCreated, &pStart, &pEnd) + switch err { + case nil: + if pTitle == o.Title && pSummary == o.Summary && pProject == o.Project && + pStart == o.LineStart && pEnd == o.LineEnd && + (o.LineEnd > 0 || pCreated == o.CreatedAt) { + return prevID, false, tx.Commit() // unchanged + } + case sql.ErrNoRows: + // first observation of this fingerprint — proceed + default: + return 0, false, fmt.Errorf("check previous observation: %w", err) + } + + res, err := tx.Exec( + `INSERT INTO observations + (fingerprint, source_id, source_path, project, content_type, title, summary, + collected_at, created_at, line_start, line_end, confidence, ingester_version, trigger, provenance) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary, + now, o.CreatedAt, o.LineStart, o.LineEnd, o.Confidence, o.IngesterVersion, o.Trigger, o.Provenance, + ) + if err != nil { + return 0, false, fmt.Errorf("insert observation: %w", err) + } + obsID, _ = res.LastInsertId() + + // Update materialized entries cache + var existingFirstSeen string + err = tx.QueryRow("SELECT first_seen FROM entries WHERE fingerprint=?", o.Fingerprint).Scan(&existingFirstSeen) + + isNew = false + if err == sql.ErrNoRows { + isNew = true + _, err = tx.Exec( + `INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?)`, + o.Fingerprint, o.SourceID, o.SourcePath, o.Project, o.ContentType, o.Title, o.Summary, + now, now, coalesceCreatedAt(o.CreatedAt), o.Confidence, + ) + } else if err == nil { + _, err = tx.Exec( + `UPDATE entries SET last_seen=?, ref_count=ref_count+1, last_confidence=?, + source_path=COALESCE(NULLIF(?,''), source_path), + title=COALESCE(NULLIF(?,''), title), + summary=COALESCE(NULLIF(?,''), summary), + created_at=COALESCE(NULLIF(?,''), created_at) + WHERE fingerprint=?`, + now, o.Confidence, o.SourcePath, o.Title, o.Summary, o.CreatedAt, o.Fingerprint, + ) + } + if err != nil { + return 0, false, fmt.Errorf("update entries cache: %w", err) + } + + // Auto-link to golden thread (only if relevant) + if goldenID, keywords, bigrams := k.goldenThreadKeywords(tx); goldenID > 0 && len(keywords) > 0 { + if o.Confidence >= goldenMinConfidence && isRelevant(o.Title, o.Summary, o.Project, keywords, bigrams) { + res, err := tx.Exec( + `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`, + goldenID, obsID, + ) + linked := err == nil && res != nil + if linked { + if n, _ := res.RowsAffected(); n == 0 { + linked = false + } + } + tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, goldenID) + + // Temporal cross-reference: when a high-signal non-browser event links, + // nearby browser history (±1h) becomes thread context too. + if linked && o.SourceID != "browser-history" && o.CreatedAt != "" { + k.linkTemporalNeighbors(tx, goldenID, o.CreatedAt, keywords, bigrams) + } + } + } + + return obsID, isNew, tx.Commit() +} + +// linkTemporalNeighbors links browser-history observations within ±1h of a +// signal timestamp to the thread. A weak keyword check still applies so +// unrelated browsing (lunch news) doesn't pollute the thread. +func (k *KnoxDB) linkTemporalNeighbors(tx *sql.Tx, threadID int64, createdAt string, keywords []string, bigrams [][2]string) { + rows, err := tx.Query( + `SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,'') + FROM observations o + WHERE o.source_id='browser-history' + AND o.created_at != '' + AND ABS(strftime('%s', o.created_at) - strftime('%s', ?)) <= 3600 + GROUP BY o.fingerprint`, + createdAt, + ) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var obsID int64 + var title, summary string + if err := rows.Scan(&obsID, &title, &summary); err != nil { + continue + } + if isRelevant(title, summary, "", keywords, bigrams) || weakRelevant(title, summary, keywords) { + tx.Exec( + `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'temporal')`, + threadID, obsID, + ) + } + } +} + +// weakRelevant is a lower bar for temporal neighbors: any single keyword match. +func weakRelevant(title, summary string, keywords []string) bool { + text := strings.ToLower(title + " " + summary) + for _, t := range wordBoundaryRE.FindAllString(text, -1) { + for _, kw := range keywords { + if t == kw { + return true + } + } + } + return false +} + +// FindEntry returns current entry from materialized cache. +func (k *KnoxDB) FindEntry(fingerprint string) (*Entry, error) { + row := k.db.QueryRow( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries WHERE fingerprint=?`, fingerprint, + ) + e := &Entry{} + err := row.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType, + &e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence) + if err == sql.ErrNoRows { + return nil, nil + } + return e, err +} + +// ObservationsByFingerprint returns the observation history for a fingerprint, +// collapsing exact duplicates (same summary/signal-time/extent) into one row +// with a Count — re-observations of unchanged content are noise, not signal. +func (k *KnoxDB) ObservationsByFingerprint(fp string, limit int) ([]Observation, error) { + if limit <= 0 { + limit = 50 + } + rows, err := k.db.Query( + `SELECT MAX(id), fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), MAX(collected_at), COALESCE(created_at,''), + COALESCE(line_start,0), COALESCE(line_end,0), confidence, ingester_version, COALESCE(trigger,''), COUNT(*) + FROM observations WHERE fingerprint=? + GROUP BY COALESCE(summary,''), COALESCE(created_at,''), COALESCE(line_end,0) + ORDER BY MAX(collected_at) DESC LIMIT ?`, fp, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var obs []Observation + for rows.Next() { + var o Observation + if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType, + &o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd, + &o.Confidence, &o.IngesterVersion, &o.Trigger, &o.Count); err != nil { + return nil, err + } + obs = append(obs, o) + } + return obs, nil +} + +// RebuildEntriesFromObservations drops and rebuilds the entries cache from observations. +// coalesceCreatedAt prefers the observation's signal time, falling back to ingestion time. +func coalesceCreatedAt(s string) string { + if s != "" { + return s + } + return time.Now().UTC().Format(time.RFC3339) +} + +func (k *KnoxDB) RebuildEntriesFromObservations() error { + tx, err := k.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if _, err := tx.Exec("DELETE FROM entries"); err != nil { + return err + } + _, err = tx.Exec( + `INSERT INTO entries (fingerprint, source_id, source_path, project, content_type, title, summary, first_seen, last_seen, created_at, ref_count, last_confidence) + SELECT + o.fingerprint, + o.source_id, + o.source_path, + o.project, + o.content_type, + o.title, + o.summary, + MIN(o.collected_at), + MAX(o.collected_at), + COALESCE(NULLIF(MAX(o.created_at),''), MIN(o.collected_at)), + COUNT(*), + MAX(o.confidence) + FROM observations o + GROUP BY o.fingerprint`, + ) + if err != nil { + return fmt.Errorf("rebuild entries: %w", err) + } + return tx.Commit() +} + +// EntriesByProject returns entries from the materialized cache. +func (k *KnoxDB) EntriesByProject(project string, limit int) ([]Entry, error) { + if limit <= 0 { + limit = 50 + } + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries WHERE project=? ORDER BY last_seen DESC LIMIT ?`, project, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +func (k *KnoxDB) RecentEntriesBySource(sourceID string, limit int) ([]Entry, error) { + if limit <= 0 { + limit = 10 + } + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries WHERE source_id=? ORDER BY last_seen DESC LIMIT ?`, sourceID, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +// EntriesBySource returns all entries for a source ordered by title (for +// stable project-status listings like git repo snapshots). +func (k *KnoxDB) EntriesBySource(sourceID string) ([]Entry, error) { + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries WHERE source_id=? ORDER BY title COLLATE NOCASE`, sourceID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +func (k *KnoxDB) RecentEntries(limit int) ([]Entry, error) { + if limit <= 0 { + limit = 20 + } + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries ORDER BY COALESCE(NULLIF(created_at,''), last_seen) DESC LIMIT ?`, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +// escapeLike escapes LIKE wildcards so user input is matched literally. +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + +func (k *KnoxDB) Search(query string, limit int) ([]Entry, error) { + if limit <= 0 { + limit = 20 + } + like := "%" + escapeLike(query) + "%" + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries + WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' + OR project LIKE ? ESCAPE '\' OR source_path LIKE ? ESCAPE '\' + ORDER BY last_seen DESC LIMIT ?`, + like, like, like, like, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +// FindEntryByPrefix returns entries whose fingerprint starts with the given prefix. +func (k *KnoxDB) FindEntryByPrefix(fp string, limit int) ([]Entry, error) { + if limit <= 0 { + limit = 5 + } + rows, err := k.db.Query( + `SELECT fingerprint, source_id, COALESCE(source_path,''), COALESCE(project,''), COALESCE(content_type,''), + COALESCE(title,''), COALESCE(summary,''), COALESCE(first_seen,''), COALESCE(last_seen,''), + COALESCE(created_at,''), ref_count, COALESCE(last_confidence,0.5) + FROM entries WHERE fingerprint LIKE ? || '%' LIMIT ?`, fp, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + return scanEntries(rows) +} + +func scanEntries(rows *sql.Rows) ([]Entry, error) { + var entries []Entry + for rows.Next() { + var e Entry + if err := rows.Scan(&e.Fingerprint, &e.SourceID, &e.SourcePath, &e.Project, &e.ContentType, + &e.Title, &e.Summary, &e.FirstSeen, &e.LastSeen, &e.CreatedAt, &e.RefCount, &e.LastConfidence); err != nil { + return nil, err + } + entries = append(entries, e) + } + return entries, nil +} + +// UpsertSession tracks session state. +func (k *KnoxDB) UpsertSession(sessionID, project, title, status string) error { + now := time.Now().UTC().Format(time.RFC3339) + _, err := k.db.Exec( + `INSERT INTO sessions (session_id, project, title, status, started_at, last_seen) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + title=COALESCE(NULLIF(?,''), title), + status=?, + last_seen=?, + agent_count=agent_count+1`, + sessionID, project, title, status, now, now, + title, status, now, + ) + return err +} + +func (k *KnoxDB) PendingSessions() ([]SessionRow, error) { + rows, err := k.db.Query( + `SELECT session_id, COALESCE(project,''), COALESCE(title,''), status, agent_count, + COALESCE(started_at,''), COALESCE(ended_at,''), last_seen, indexed + FROM sessions WHERE indexed=0 ORDER BY last_seen DESC`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var sessions []SessionRow + for rows.Next() { + var s SessionRow + if err := rows.Scan(&s.SessionID, &s.Project, &s.Title, &s.Status, &s.AgentCount, + &s.StartedAt, &s.EndedAt, &s.LastSeen, &s.Indexed); err != nil { + return nil, err + } + sessions = append(sessions, s) + } + return sessions, nil +} + +func (k *KnoxDB) MarkSessionIndexed(sessionID string) error { + _, err := k.db.Exec("UPDATE sessions SET indexed=1 WHERE session_id=?", sessionID) + return err +} + +func (k *KnoxDB) Stats() (map[string]any, error) { + stats := make(map[string]any) + + var v int + var s string + + k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&v) + stats["total_entries"] = v + v = 0 + k.db.QueryRow("SELECT COUNT(*) FROM entries WHERE last_seen > datetime('now', '-1 day')").Scan(&v) + stats["entries_last_24h"] = v + v = 0 + k.db.QueryRow("SELECT COUNT(DISTINCT project) FROM entries").Scan(&v) + stats["total_projects"] = v + v = 0 + k.db.QueryRow("SELECT COUNT(*) FROM observations").Scan(&v) + stats["total_observations"] = v + v = 0 + k.db.QueryRow("SELECT COUNT(*) FROM observations WHERE collected_at > datetime('now', '-1 day')").Scan(&v) + stats["observations_last_24h"] = v + v = 0 + k.db.QueryRow("SELECT COALESCE(MIN(collected_at),'') FROM observations").Scan(&s) + stats["earliest_observation"] = s + s = "" + k.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&v) + stats["total_sessions"] = v + v = 0 + k.db.QueryRow("SELECT COUNT(*) FROM sessions WHERE indexed=0").Scan(&v) + stats["pending_reflections"] = v + v = 0 + + goldenID, _ := k.GoldenThreadID() + if goldenID > 0 { + stats["golden_thread_id"] = goldenID + t, _ := k.GetThread(goldenID) + if t != nil { + stats["golden_thread"] = t.Title + } + } + + return stats, nil +} + +// ProvenanceChain returns the provenance link graph for a fingerprint. +func (k *KnoxDB) ProvenanceChain(fp string, maxHops int) ([]struct { + Fingerprint string + Relation string + HopDistance int +}, error) { + if maxHops <= 0 { + maxHops = 3 + } + rows, err := k.db.Query( + `SELECT ancestor_fp, relation, hop_distance FROM provenance_links + WHERE descendant_fp=? AND hop_distance <= ? + UNION + SELECT descendant_fp, relation, hop_distance FROM provenance_links + WHERE ancestor_fp=? AND hop_distance <= ? + ORDER BY hop_distance`, fp, maxHops, fp, maxHops, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []struct { + Fingerprint string + Relation string + HopDistance int + } + for rows.Next() { + var r struct { + Fingerprint string + Relation string + HopDistance int + } + if err := rows.Scan(&r.Fingerprint, &r.Relation, &r.HopDistance); err != nil { + return nil, err + } + results = append(results, r) + } + return results, nil +} + +// ─── Golden Thread ─────────────────────────────────────────── + +func (k *KnoxDB) goldenThreadID(tx *sql.Tx) int64 { + var id int64 + err := tx.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id) + if err != nil { + return 0 + } + return id +} + +func (k *KnoxDB) GoldenThreadID() (int64, error) { + var id int64 + err := k.db.QueryRow("SELECT value FROM settings WHERE key='golden_thread_id'").Scan(&id) + if err == sql.ErrNoRows { + return 0, nil + } + return id, err +} + +func (k *KnoxDB) SetGoldenThread(id int64) error { + if id == 0 { + _, err := k.db.Exec("DELETE FROM settings WHERE key='golden_thread_id'") + return err + } + // Verify thread exists + t, err := k.GetThread(id) + if err != nil || t == nil { + return fmt.Errorf("thread #%d not found", id) + } + _, err = k.db.Exec( + "INSERT INTO settings (key, value) VALUES ('golden_thread_id', ?) ON CONFLICT(key) DO UPDATE SET value=?", + id, id, + ) + return err +} + +// kwCache caches DF-filtered golden-thread keywords, keyed by thread update time. +var kwCache = struct { + sync.Mutex + threadID int64 + updatedAt string + keywords []string + bigrams [][2]string +}{threadID: -1} + +// maxDocFreq is the fraction of entries a keyword may match before it's +// considered too generic to discriminate (e.g. "board"). +const maxDocFreq = 0.02 + +func (k *KnoxDB) goldenThreadKeywords(tx *sql.Tx) (int64, []string, [][2]string) { + var id int64 + var title, motivation, tags, updatedAt string + err := tx.QueryRow( + `SELECT t.id, t.title, COALESCE(t.motivation,''), COALESCE(t.tags,'[]'), COALESCE(t.updated_at,'') + FROM threads t JOIN settings s ON s.value = CAST(t.id AS TEXT) + WHERE s.key='golden_thread_id'`, + ).Scan(&id, &title, &motivation, &tags, &updatedAt) + if err != nil { + return 0, nil, nil + } + + kwCache.Lock() + if kwCache.threadID == id && kwCache.updatedAt == updatedAt { + kws, bgs := kwCache.keywords, kwCache.bigrams + kwCache.Unlock() + return id, kws, bgs + } + kwCache.Unlock() + + raw := extractKeywords(title + " " + motivation + " " + tags) + filtered := k.filterByDocFreq(tx, raw) + + // Keep only bigrams where both words survived DF filtering. + survived := make(map[string]bool, len(filtered)) + for _, kw := range filtered { + survived[kw] = true + } + var bigrams [][2]string + for _, bg := range extractBigrams(title + " " + motivation + " " + tags) { + if survived[bg[0]] && survived[bg[1]] { + bigrams = append(bigrams, bg) + } + } + + kwCache.Lock() + kwCache.threadID, kwCache.updatedAt = id, updatedAt + kwCache.keywords, kwCache.bigrams = filtered, bigrams + kwCache.Unlock() + return id, filtered, bigrams +} + +// filterByDocFreq drops keywords that match more than maxDocFreq of all entries. +// Rare keywords (klubhaus, doorbell) are discriminative; common ones (board, +// current) just link noise. +func (k *KnoxDB) filterByDocFreq(tx *sql.Tx, keywords []string) []string { + var total int + if err := tx.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 { + return keywords + } + limit := int(float64(total) * maxDocFreq) + if limit < 5 { + limit = 5 + } + var out []string + for _, kw := range keywords { + var df int + like := "%" + escapeLike(kw) + "%" + err := tx.QueryRow( + `SELECT COUNT(*) FROM entries + WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`, + like, like, like, + ).Scan(&df) + if err != nil || df <= limit { + out = append(out, kw) + } + } + return out +} + +var wordRE = regexp.MustCompile(`[^a-z0-9]+`) + +// wordBoundaryRE extracts maximal alphanumeric runs. No \b anchors — Go treats +// underscore as a word char, so \b would swallow snake_case tokens entirely +// ("ALERT_klubhaus_topic" would yield nothing). +var wordBoundaryRE = regexp.MustCompile(`[a-z0-9]+`) + +var stopWords = map[string]bool{ + "with": true, "this": true, "that": true, "from": true, + "they": true, "have": true, "were": true, "what": true, + "when": true, "where": true, "which": true, "their": true, + "them": true, "been": true, "also": true, "than": true, + "into": true, "over": true, "such": true, "each": true, + "about": true, "most": true, "some": true, "more": true, + "other": true, "then": true, "will": true, "just": true, + "like": true, "only": true, "very": true, "even": true, + "much": true, "still": true, "well": true, "here": true, + "there": true, "does": true, "done": true, "system": true, + "page": true, "site": true, "user": true, "data": true, + "file": true, "type": true, "home": true, "mode": true, + "show": true, "name": true, "back": true, "open": true, +} + +func ExtractKeywords(s string) []string { return extractKeywords(s) } +func extractKeywords(s string) []string { + s = strings.ToLower(s) + words := wordRE.Split(s, -1) + seen := make(map[string]bool) + var keywords []string + for _, w := range words { + if len(w) > 3 && !seen[w] && !stopWords[w] { + seen[w] = true + keywords = append(keywords, w) + } + } + return keywords +} + +// extractBigrams returns ordered consecutive keyword pairs from the text, +// preserving phrase structure ("multi board" from "multi-board"). +func extractBigrams(s string) [][2]string { + s = strings.ToLower(s) + words := wordRE.Split(s, -1) + var kept []string + for _, w := range words { + if len(w) > 3 && !stopWords[w] { + kept = append(kept, w) + } + } + seen := make(map[[2]string]bool) + var out [][2]string + for i := 0; i+1 < len(kept); i++ { + pair := [2]string{kept[i], kept[i+1]} + if !seen[pair] { + seen[pair] = true + out = append(out, pair) + } + } + return out +} + +// isRelevant scores phrase and keyword overlap: +// - a bigram phrase match (in title or body) scores 3 — "multi board" is +// precise, "board" alone is not +// - a keyword in the title scores 2 +// - a keyword in summary/project scores 1 +// +// Threshold is 3, so a single weak keyword never links by itself. +func isRelevant(title, summary, project string, keywords []string, bigrams [][2]string) bool { + titleTokens := wordBoundaryRE.FindAllString(strings.ToLower(title), -1) + bodyTokens := wordBoundaryRE.FindAllString(strings.ToLower(summary+" "+project), -1) + + titleSet := make(map[string]bool, len(titleTokens)) + for _, t := range titleTokens { + titleSet[t] = true + } + bodySet := make(map[string]bool, len(bodyTokens)) + for _, t := range bodyTokens { + bodySet[t] = true + } + + score := 0 + for _, kw := range keywords { + if titleSet[kw] { + score += 2 + } else if bodySet[kw] { + score += 1 + } + } + for _, bg := range bigrams { + if containsBigram(titleTokens, bg) || containsBigram(bodyTokens, bg) { + score += 3 + } + } + return score >= 3 +} + +func containsBigram(tokens []string, bg [2]string) bool { + for i := 0; i+1 < len(tokens); i++ { + if tokens[i] == bg[0] && tokens[i+1] == bg[1] { + return true + } + } + return false +} + +// ─── Thread (Motivation/Intent) Operations ───────────────────── + +type Thread struct { + ID int64 + Title string + Motivation string + Status string + Priority string + CreatedAt string + UpdatedAt string + ResolvedAt string + Tags string + Provenance string + EntryCount int +} + +func ThreadFP(id int64) string { return fmt.Sprintf("thread:%d", id) } + +func (k *KnoxDB) CreateThread(title, motivation, priority, tags, provenance string) (int64, error) { + res, err := k.db.Exec( + `INSERT INTO threads (title, motivation, priority, tags, provenance) VALUES (?, ?, ?, ?, ?)`, + title, motivation, priority, tags, provenance, + ) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func (k *KnoxDB) ListThreads(status string) ([]Thread, error) { + query := `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, + t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), + COALESCE(t.provenance,'{}'), + (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) + FROM threads t` + var args []any + if status != "" { + query += " WHERE t.status=?" + args = append(args, status) + } + query += " ORDER BY t.updated_at DESC" + + rows, err := k.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var threads []Thread + for rows.Next() { + var t Thread + if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, + &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil { + return nil, err + } + threads = append(threads, t) + } + return threads, nil +} + +func (k *KnoxDB) GetThread(id int64) (*Thread, error) { + row := k.db.QueryRow( + `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, + t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), + COALESCE(t.provenance,'{}'), + (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) + FROM threads t WHERE t.id=?`, id, + ) + t := &Thread{} + err := row.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, + &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount) + if err == sql.ErrNoRows { + return nil, nil + } + return t, err +} + +func (k *KnoxDB) CloseThread(id int64) error { + _, err := k.db.Exec( + `UPDATE threads SET status='resolved', resolved_at=datetime('now'), updated_at=datetime('now') WHERE id=?`, + id, + ) + return err +} + +// UpdateThread rewrites the editorial fields of a thread (used by the LLM +// draft/polish pass to replace auto-generated titles/motivations). Empty +// strings leave the corresponding field unchanged; omitting them all is a no-op. +func (k *KnoxDB) UpdateThread(id int64, title, motivation, priority, tags string) (bool, error) { + t, err := k.GetThread(id) + if err != nil || t == nil { + return false, fmt.Errorf("thread #%d not found", id) + } + if title == "" { + title = t.Title + } + if motivation == "" { + motivation = t.Motivation + } + if priority == "" { + priority = t.Priority + } + if tags == "" { + tags = t.Tags + } + changed := title != t.Title || motivation != t.Motivation || priority != t.Priority || tags != t.Tags + if !changed { + return false, nil + } + _, err = k.db.Exec( + `UPDATE threads SET title=?, motivation=?, priority=?, tags=?, updated_at=datetime('now') WHERE id=?`, + title, motivation, priority, tags, id, + ) + return true, err +} + +func (k *KnoxDB) LinkObservationToThread(threadID, observationID int64, relevance string) error { + _, err := k.db.Exec( + `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, ?)`, + threadID, observationID, relevance, + ) + if err != nil { + return err + } + _, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID) + return err +} + +// AutoLinkThreadObservations resolves each entry fingerprint to its most recent +// observation and links it to the thread ('auto_cluster' relevance). Idempotent: +// INSERT OR IGNORE means re-linking an already-linked fingerprint is a no-op. +// Returns the number of newly linked observations. +func (k *KnoxDB) AutoLinkThreadObservations(threadID int64, fingerprints []string) (int, error) { + tx, err := k.db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + + linked := 0 + for _, fp := range fingerprints { + if fp == "" { + continue + } + var obsID int64 + if err := tx.QueryRow( + `SELECT id FROM observations WHERE fingerprint=? ORDER BY id DESC LIMIT 1`, fp, + ).Scan(&obsID); err != nil { + continue + } + res, err := tx.Exec( + `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto_cluster')`, + threadID, obsID, + ) + if err != nil { + continue + } + if n, _ := res.RowsAffected(); n > 0 { + linked++ + } + } + + if linked > 0 { + if _, err := tx.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID); err != nil { + return linked, err + } + } + return linked, tx.Commit() +} + +// ActiveThreadByKeyword returns the most recently updated active thread whose +// title/motivation/tags contains the given keyword, or 0 if none exists. Used +// to fold new clusters into an existing thread instead of spawning duplicates. +func (k *KnoxDB) ActiveThreadByKeyword(keyword string) (int64, error) { + if keyword == "" { + return 0, nil + } + like := "%" + escapeLike(keyword) + "%" + rows, err := k.db.Query( + `SELECT id FROM threads + WHERE status='active' AND (title LIKE ? ESCAPE '\' OR motivation LIKE ? ESCAPE '\' OR tags LIKE ? ESCAPE '\') + ORDER BY updated_at DESC LIMIT 1`, + like, like, like, + ) + if err != nil { + return 0, err + } + defer rows.Close() + if rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return 0, err + } + return id, nil + } + return 0, rows.Err() +} + +// ThreadObservations returns linked observations, deduplicated by fingerprint +// (re-observed entries appear once, with the latest observation's data). +func (k *KnoxDB) ThreadObservations(threadID int64) ([]Observation, error) { + rows, err := k.db.Query( + `SELECT o.id, o.fingerprint, o.source_id, COALESCE(o.source_path,''), COALESCE(o.project,''), + COALESCE(o.content_type,''), COALESCE(o.title,''), COALESCE(o.summary,''), + o.collected_at, COALESCE(o.created_at,''), COALESCE(o.line_start,0), COALESCE(o.line_end,0), + o.confidence, o.ingester_version, COALESCE(o.trigger,'') + FROM observations o + JOIN thread_observations to2 ON to2.observation_id = o.id + WHERE to2.thread_id=? + AND o.id IN ( + SELECT MAX(to3.observation_id) FROM thread_observations to3 + JOIN observations o2 ON o2.id = to3.observation_id + WHERE to3.thread_id=? + GROUP BY o2.fingerprint + ) + ORDER BY COALESCE(NULLIF(o.created_at,''), o.collected_at) DESC`, threadID, threadID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var obs []Observation + for rows.Next() { + var o Observation + if err := rows.Scan(&o.ID, &o.Fingerprint, &o.SourceID, &o.SourcePath, &o.Project, &o.ContentType, + &o.Title, &o.Summary, &o.CollectedAt, &o.CreatedAt, &o.LineStart, &o.LineEnd, + &o.Confidence, &o.IngesterVersion, &o.Trigger); err != nil { + return nil, err + } + obs = append(obs, o) + } + return obs, nil +} + +func (k *KnoxDB) AddThreadNote(threadID int64, note string) (int64, error) { + res, err := k.db.Exec( + `INSERT INTO thread_notes (thread_id, note) VALUES (?, ?)`, + threadID, note, + ) + if err != nil { + return 0, err + } + _, err = k.db.Exec(`UPDATE threads SET updated_at=datetime('now') WHERE id=?`, threadID) + return res.LastInsertId() +} + +func (k *KnoxDB) ThreadNotes(threadID int64) ([]struct { + ID int64 + Note string + CreatedAt string +}, error) { + rows, err := k.db.Query( + `SELECT id, note, created_at FROM thread_notes WHERE thread_id=? ORDER BY created_at`, threadID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var notes []struct { + ID int64 + Note string + CreatedAt string + } + for rows.Next() { + var n struct { + ID int64 + Note string + CreatedAt string + } + if err := rows.Scan(&n.ID, &n.Note, &n.CreatedAt); err != nil { + return nil, err + } + notes = append(notes, n) + } + return notes, nil +} + +func (k *KnoxDB) SearchThreadsByMotivation(query string) ([]Thread, error) { + like := "%" + escapeLike(query) + "%" + rows, err := k.db.Query( + `SELECT t.id, t.title, COALESCE(t.motivation,''), t.status, t.priority, + t.created_at, t.updated_at, COALESCE(t.resolved_at,''), COALESCE(t.tags,'[]'), + COALESCE(t.provenance,'{}'), + (SELECT COUNT(*) FROM thread_observations WHERE thread_id=t.id) + FROM threads t + WHERE t.title LIKE ? ESCAPE '\' OR t.motivation LIKE ? ESCAPE '\' OR t.tags LIKE ? ESCAPE '\' + ORDER BY t.updated_at DESC LIMIT 20`, + like, like, like, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var threads []Thread + for rows.Next() { + var t Thread + if err := rows.Scan(&t.ID, &t.Title, &t.Motivation, &t.Status, &t.Priority, + &t.CreatedAt, &t.UpdatedAt, &t.ResolvedAt, &t.Tags, &t.Provenance, &t.EntryCount); err != nil { + return nil, err + } + threads = append(threads, t) + } + return threads, nil +} + +// LinkEntryToThread links an entry to a thread via provenance_links. +func (k *KnoxDB) LinkEntryToThread(threadID int64, entryFP, relation string) error { + if entryFP == "" { + return fmt.Errorf("entry fingerprint must not be empty") + } + tfp := ThreadFP(threadID) + _, err := k.db.Exec( + `INSERT OR IGNORE INTO provenance_links (descendant_fp, ancestor_fp, hop_distance, relation) VALUES (?, ?, 1, ?)`, + entryFP, tfp, relation, + ) + return err +} + +// ThreadRelevanceProfile returns the DF-filtered keywords and bigrams for any thread. +func (k *KnoxDB) ThreadRelevanceProfile(threadID int64) ([]string, [][2]string, error) { + var title, motivation, tags string + err := k.db.QueryRow( + `SELECT title, COALESCE(motivation,''), COALESCE(tags,'[]') FROM threads WHERE id=?`, threadID, + ).Scan(&title, &motivation, &tags) + if err != nil { + return nil, nil, err + } + raw := extractKeywords(title + " " + motivation + " " + tags) + + var total int + if err := k.db.QueryRow("SELECT COUNT(*) FROM entries").Scan(&total); err != nil || total == 0 { + return raw, nil, nil + } + limit := int(float64(total) * maxDocFreq) + if limit < 5 { + limit = 5 + } + survived := make(map[string]bool) + var keywords []string + for _, kw := range raw { + var df int + like := "%" + escapeLike(kw) + "%" + if err := k.db.QueryRow( + `SELECT COUNT(*) FROM entries + WHERE title LIKE ? ESCAPE '\' OR summary LIKE ? ESCAPE '\' OR project LIKE ? ESCAPE '\'`, + like, like, like, + ).Scan(&df); err != nil || df <= limit { + keywords = append(keywords, kw) + survived[kw] = true + } + } + var bigrams [][2]string + for _, bg := range extractBigrams(title + " " + motivation + " " + tags) { + if survived[bg[0]] && survived[bg[1]] { + bigrams = append(bigrams, bg) + } + } + return keywords, bigrams, nil +} + +// PruneThread removes machine-linked observations ('auto' and legacy 'regroom') +// that fail the (stricter) relevance scoring, using the thread's DF-filtered +// keyword profile. Temporal links and manual links (descriptive relevance +// strings) are evidence of a different kind and are left intact. +func (k *KnoxDB) PruneThread(threadID int64) (int, error) { + keywords, bigrams, err := k.ThreadRelevanceProfile(threadID) + if err != nil { + return 0, err + } + if len(keywords) == 0 { + return 0, nil + } + rows, err := k.db.Query( + `SELECT to2.observation_id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'') + FROM thread_observations to2 + JOIN observations o ON o.id = to2.observation_id + WHERE to2.thread_id = ? AND to2.relevance IN ('auto', 'regroom')`, + threadID, + ) + if err != nil { + return 0, err + } + + var staleIDs []int64 + for rows.Next() { + var id int64 + var title, summary, project string + if err := rows.Scan(&id, &title, &summary, &project); err != nil { + rows.Close() + return 0, err + } + if !isRelevant(title, summary, project, keywords, bigrams) { + staleIDs = append(staleIDs, id) + } + } + rows.Close() + + if len(staleIDs) == 0 { + return 0, nil + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(staleIDs)), ",") + args := make([]any, 0, len(staleIDs)+1) + args = append(args, threadID) + for _, id := range staleIDs { + args = append(args, id) + } + res, err := k.db.Exec( + fmt.Sprintf(`DELETE FROM thread_observations WHERE thread_id=? AND observation_id IN (%s)`, placeholders), + args..., + ) + if err != nil { + return 0, err + } + n, _ := res.RowsAffected() + return int(n), nil +} + +// RescanThread links existing observations that pass the thread's relevance +// profile but were never linked — the inverse of PruneThread. One link row per +// fingerprint (latest observation), relevance 'auto'. +func (k *KnoxDB) RescanThread(threadID int64) (int, error) { + keywords, bigrams, err := k.ThreadRelevanceProfile(threadID) + if err != nil { + return 0, err + } + if len(keywords) == 0 { + return 0, nil + } + rows, err := k.db.Query( + `SELECT o.id, COALESCE(o.title,''), COALESCE(o.summary,''), COALESCE(o.project,'') + FROM observations o + WHERE o.id IN (SELECT MAX(id) FROM observations GROUP BY fingerprint) + AND o.fingerprint NOT IN ( + SELECT o2.fingerprint FROM thread_observations t + JOIN observations o2 ON o2.id = t.observation_id WHERE t.thread_id = ? + )`, + threadID, + ) + if err != nil { + return 0, err + } + + var linkIDs []int64 + for rows.Next() { + var id int64 + var title, summary, project string + if err := rows.Scan(&id, &title, &summary, &project); err != nil { + rows.Close() + return 0, err + } + if isRelevant(title, summary, project, keywords, bigrams) { + linkIDs = append(linkIDs, id) + } + } + rows.Close() + + linked := 0 + for _, id := range linkIDs { + res, err := k.db.Exec( + `INSERT OR IGNORE INTO thread_observations (thread_id, observation_id, relevance) VALUES (?, ?, 'auto')`, + threadID, id, + ) + if err == nil { + if n, _ := res.RowsAffected(); n > 0 { + linked++ + } + } + } + return linked, nil +} + +// ThreadProvenance returns the provenance graph for a thread. +func (k *KnoxDB) ThreadProvenance(threadID int64) ([]struct { + Fingerprint string + Relation string + HopDistance int +}, error) { + return k.ProvenanceChain(ThreadFP(threadID), 3) +} diff --git a/internal/db/schema.go b/internal/db/schema.go new file mode 100644 index 0000000..06b1c7d --- /dev/null +++ b/internal/db/schema.go @@ -0,0 +1,135 @@ +package db + +const Schema = ` +-- IMMUTABLE LOG: every observation is append-only, never mutated +CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + fingerprint TEXT NOT NULL, + source_id TEXT NOT NULL, + source_path TEXT, + project TEXT, + content_type TEXT, + title TEXT, + summary TEXT, + collected_at TEXT NOT NULL DEFAULT (datetime('now')), + created_at TEXT, + line_start INTEGER, + line_end INTEGER, + confidence REAL DEFAULT 0.5, + ingester_version TEXT DEFAULT '1', + trigger TEXT, + provenance TEXT DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_obs_fingerprint ON observations(fingerprint); +CREATE INDEX IF NOT EXISTS idx_obs_collected ON observations(collected_at); +CREATE INDEX IF NOT EXISTS idx_obs_created ON observations(created_at); +CREATE INDEX IF NOT EXISTS idx_obs_source ON observations(source_id, source_path); +CREATE INDEX IF NOT EXISTS idx_obs_project ON observations(project); +CREATE INDEX IF NOT EXISTS idx_obs_trigger ON observations(trigger); + +-- MATERIALIZED CACHE: current state, reconstructable from observations +CREATE TABLE IF NOT EXISTS entries ( + fingerprint TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + source_path TEXT, + project TEXT, + content_type TEXT, + title TEXT, + summary TEXT, + first_seen TEXT, + last_seen TEXT, + ref_count INTEGER DEFAULT 1, + last_confidence REAL DEFAULT 0.5 +); + +CREATE INDEX IF NOT EXISTS idx_entries_project ON entries(project); +CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source_id); +CREATE INDEX IF NOT EXISTS idx_entries_seen ON entries(last_seen); + +-- PROVENANCE LINKS: multi-hop lineage between entries +CREATE TABLE IF NOT EXISTS provenance_links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + descendant_fp TEXT NOT NULL, + ancestor_fp TEXT NOT NULL, + hop_distance INTEGER NOT NULL, + relation TEXT NOT NULL, + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_pl_descendant ON provenance_links(descendant_fp); +CREATE INDEX IF NOT EXISTS idx_pl_ancestor ON provenance_links(ancestor_fp); + +-- REFLECTION STATE: per-session tracking +CREATE TABLE IF NOT EXISTS sessions ( + session_id TEXT PRIMARY KEY, + project TEXT, + title TEXT, + status TEXT DEFAULT 'active', + agent_count INTEGER DEFAULT 0, + started_at TEXT, + ended_at TEXT, + last_seen TEXT DEFAULT (datetime('now')), + indexed INTEGER DEFAULT 0 +); + +-- SOURCE REGISTRY +CREATE TABLE IF NOT EXISTS sources ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + enabled INTEGER DEFAULT 1, + last_event TEXT, + created_at TEXT DEFAULT (datetime('now')) +); + +INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-session', 'Opencode Session Diffs'); +INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-log', 'Opencode Log Files'); +INSERT OR IGNORE INTO sources (id, name) VALUES ('git', 'Git History'); +INSERT OR IGNORE INTO sources (id, name) VALUES ('filesystem', 'File System Watcher'); + +-- KNOWLEDGE THREADS: why observations exist, what questions they answer +CREATE TABLE IF NOT EXISTS threads ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + motivation TEXT, + status TEXT DEFAULT 'active', + priority TEXT DEFAULT 'medium', + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')), + resolved_at TEXT, + tags TEXT DEFAULT '[]', + provenance TEXT DEFAULT '{}' +); + +-- SETTINGS: key-value store for runtime configuration +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT +); + +CREATE INDEX IF NOT EXISTS idx_threads_status ON threads(status); + +CREATE TABLE IF NOT EXISTS thread_observations ( + thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE, + observation_id INTEGER NOT NULL REFERENCES observations(id) ON DELETE CASCADE, + relevance TEXT, + added_at TEXT DEFAULT (datetime('now')), + PRIMARY KEY (thread_id, observation_id) +); + +CREATE TABLE IF NOT EXISTS thread_notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE, + note TEXT NOT NULL, + created_at TEXT DEFAULT (datetime('now')) +); + +-- TOPICS: TF-IDF clustered groups of related observations +CREATE TABLE IF NOT EXISTS topics ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + keywords TEXT, + score REAL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')) +); +` diff --git a/internal/index/index.go b/internal/index/index.go new file mode 100644 index 0000000..c69d939 --- /dev/null +++ b/internal/index/index.go @@ -0,0 +1,100 @@ +package index + +import ( + "fmt" + "strings" + + "github.com/david/knox/internal/db" +) + +type Reflector struct { + db *db.KnoxDB +} + +func NewReflector(kdb *db.KnoxDB) *Reflector { + return &Reflector{db: kdb} +} + +type ReflectionResult struct { + SessionID string + Project string + EntryCount int + Status string +} + +func (r *Reflector) Reflect() ([]ReflectionResult, error) { + sessions, err := r.db.PendingSessions() + if err != nil { + return nil, fmt.Errorf("pending sessions: %w", err) + } + + var results []ReflectionResult + for _, s := range sessions { + entries, err := r.db.EntriesByProject(s.Project, 100) + if err != nil { + return nil, err + } + + result := ReflectionResult{ + SessionID: s.SessionID, + Project: s.Project, + EntryCount: len(entries), + Status: "reflected", + } + + if err := r.db.MarkSessionIndexed(s.SessionID); err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil +} + +type GapReport struct { + Project string + TotalEntries int +} + +func (r *Reflector) Gaps() ([]GapReport, error) { + entries, err := r.db.RecentEntries(500) + if err != nil { + return nil, err + } + + projects := make(map[string]int) + for _, e := range entries { + if e.Project != "" { + projects[e.Project]++ + } + } + + var reports []GapReport + for proj, count := range projects { + reports = append(reports, GapReport{ + Project: proj, + TotalEntries: count, + }) + } + return reports, nil +} + +func (r *Reflector) FormatResult(results []ReflectionResult) string { + var b strings.Builder + for _, res := range results { + b.WriteString(fmt.Sprintf(" %-12s %-20s %d entries [%s]\n", + truncateStr(res.SessionID, 12), + res.Project, + res.EntryCount, + res.Status, + )) + } + return b.String() +} + +func truncateStr(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) + "..." +} diff --git a/internal/index/tfidf.go b/internal/index/tfidf.go new file mode 100644 index 0000000..85c2482 --- /dev/null +++ b/internal/index/tfidf.go @@ -0,0 +1,258 @@ +package index + +import ( + "math" + "regexp" + "sort" + "strings" + + "github.com/david/knox/internal/db" +) + +type TopicCluster struct { + Name string + Keywords []string + Score float64 + Entries []db.Entry + BySource map[string]int +} + +var tokenRE = regexp.MustCompile(`[a-z]{4,}`) +var hexTokenRE = regexp.MustCompile(`^[a-f0-9]{8,}$`) + +func Tokenize(text string) []string { return tokenize(text) } +func tokenize(text string) []string { + text = strings.ToLower(text) + text = strings.ReplaceAll(text, "-", " ") + text = strings.ReplaceAll(text, "_", " ") + return tokenRE.FindAllString(text, -1) +} + +type document struct { + id int + title string + text string + source string + entry db.Entry +} + +// TF-IDF index built from all observations +type TFIDFIndex struct { + docCount int + docFreq map[string]int // term → how many docs contain it + termDocs []map[string]float64 // per-doc: term → TF score + documents []document +} + +func BuildTFIDF(entries []db.Entry) *TFIDFIndex { + idx := &TFIDFIndex{ + docFreq: make(map[string]int), + termDocs: make([]map[string]float64, 0, len(entries)), + } + + for _, e := range entries { + text := e.Title + " " + e.Summary + " " + e.SourceID + " " + e.Project + tokens := tokenize(text) + if len(tokens) == 0 { + continue + } + + doc := document{ + id: idx.docCount, + title: e.Title, + text: text, + source: e.SourceID, + entry: e, + } + + // Term frequency in this doc + tf := make(map[string]float64) + seen := make(map[string]bool) + for _, t := range tokens { + if hexTokenRE.MatchString(t) || len(t) > 25 { + continue + } + tf[t]++ + if !seen[t] { + idx.docFreq[t]++ + seen[t] = true + } + } + // Normalize TF by doc length + docLen := float64(len(tokens)) + for t, c := range tf { + tf[t] = c / docLen + } + + idx.termDocs = append(idx.termDocs, tf) + idx.documents = append(idx.documents, doc) + idx.docCount++ + } + + return idx +} + +func (idx *TFIDFIndex) TFIDF(term string, docIdx int) float64 { + tf, ok := idx.termDocs[docIdx][term] + if !ok { + return 0 + } + df := idx.docFreq[term] + if df == 0 { + return 0 + } + idf := math.Log(float64(idx.docCount) / float64(df)) + return tf * idf +} + +// TopTerms returns the N highest TF-IDF scoring terms for a document +func (idx *TFIDFIndex) TopTerms(docIdx int, n int) []struct { + Term string + Score float64 +} { + var scored []struct { + Term string + Score float64 + } + for t := range idx.termDocs[docIdx] { + s := idx.TFIDF(t, docIdx) + if s > 0 { + scored = append(scored, struct { + Term string + Score float64 + }{t, s}) + } + } + sort.Slice(scored, func(i, j int) bool { return scored[i].Score > scored[j].Score }) + if len(scored) > n { + scored = scored[:n] + } + return scored +} + +// Cluster groups observations into topics by shared top TF-IDF terms +func (idx *TFIDFIndex) Cluster(minSharedTerms int, maxTopics int) []TopicCluster { + if maxTopics <= 0 { + maxTopics = 20 + } + if minSharedTerms <= 0 { + minSharedTerms = 2 + } + + // Get top 5 terms per doc + type docTerms struct { + docIdx int + terms []string + } + var docTermList []docTerms + for i := 0; i < idx.docCount; i++ { + tt := idx.TopTerms(i, 5) + if len(tt) >= minSharedTerms { + terms := make([]string, len(tt)) + for j, t := range tt { + terms[j] = t.Term + } + docTermList = append(docTermList, docTerms{i, terms}) + } + } + + // Greedy clustering: docs sharing >= minSharedTerms terms become a topic + var clusters []TopicCluster + assigned := make(map[int]bool) + + for _, dt := range docTermList { + if assigned[dt.docIdx] { + continue + } + + cluster := TopicCluster{ + Keywords: dt.terms, + Score: 0, + BySource: make(map[string]int), + } + + // The seed doc belongs to this cluster too. + if seed := idx.documents[dt.docIdx]; seed.source != "" { + cluster.BySource[seed.source]++ + cluster.Entries = append(cluster.Entries, seed.entry) + } + + // Find all docs sharing terms with this seed + for _, other := range docTermList { + if assigned[other.docIdx] { + continue + } + shared := 0 + for _, t1 := range dt.terms { + for _, t2 := range other.terms { + if t1 == t2 { + shared++ + break + } + } + } + if shared >= minSharedTerms { + assigned[other.docIdx] = true + e := idx.documents[other.docIdx] + cluster.BySource[e.source]++ + cluster.Entries = append(cluster.Entries, e.entry) + + // Score is sum of TF-IDF of shared terms + for _, t := range dt.terms { + cluster.Score += idx.TFIDF(t, other.docIdx) + } + } + } + + if len(cluster.BySource) > 0 { + // Name the topic by top scoring TF-IDF terms across all docs + allTerms := make(map[string]float64) + for _, t := range dt.terms { + for i := 0; i < idx.docCount; i++ { + allTerms[t] += idx.TFIDF(t, i) + } + } + type kv struct { + k string + v float64 + } + var sorted []kv + for k, v := range allTerms { + sorted = append(sorted, kv{k, v}) + } + sort.Slice(sorted, func(i, j int) bool { return sorted[i].v > sorted[j].v }) + + var nameParts []string + for _, kv := range sorted { + if len(nameParts) >= 3 { + break + } + nameParts = append(nameParts, kv.k) + } + cluster.Name = strings.Join(nameParts, " / ") + + clusters = append(clusters, cluster) + if len(clusters) >= maxTopics { + break + } + } + } + + return clusters +} + +// Gaps finds topics with high browser activity but no skill coverage +func (idx *TFIDFIndex) Gaps(clusters []TopicCluster, skillKeywords map[string]bool) []TopicCluster { + var gaps []TopicCluster + for _, c := range clusters { + browserScore := c.BySource["browser-history"] + skillScore := c.BySource["skills-catalog"] + if browserScore > 0 && skillScore == 0 { + gaps = append(gaps, c) + } + } + sort.Slice(gaps, func(i, j int) bool { + return gaps[i].BySource["browser-history"] > gaps[j].BySource["browser-history"] + }) + return gaps +} diff --git a/internal/ingest/browser.go b/internal/ingest/browser.go new file mode 100644 index 0000000..ca6fa28 --- /dev/null +++ b/internal/ingest/browser.go @@ -0,0 +1,257 @@ +package ingest + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +const ( + chromiumHistoryPath = ".config/chromium/Default/History" + minHistoryPath = ".config/Min/BrowserHistory.db" +) + +// noiseURLPatterns are URLs that represent navigation noise rather than knowledge. +var noiseURLPatterns = []string{ + "opencode.ai", + "auth.opencode.ai", + "github.com/login/oauth/authorize", + "accounts.google.com", + "google.com/accounts", + "kagi.com", + "chatgpt.com", + // knox indexing its own web UI is self-referential noise + "localhost:8924", + "localhost:18925", + "127.0.0.1:8924", + "127.0.0.1:18925", +} + +func noiseURL(url string) bool { + for _, p := range noiseURLPatterns { + if strings.Contains(url, p) { + return true + } + } + return false +} + +type BrowserHistoryIngester struct { + Version string +} + +func NewBrowserHistoryIngester() *BrowserHistoryIngester { + return &BrowserHistoryIngester{Version: "browser-history/v1"} +} + +func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" } + +type historyRow struct { + URL string + Title string + VisitTime time.Time + VisitCount int +} + +func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) { + home, _ := os.UserHomeDir() + var results []*IngestResult + + // Chromium + chromiumDB := filepath.Join(home, chromiumHistoryPath) + cr, err := b.ingestChromium(chromiumDB) + if err == nil { + results = append(results, cr...) + } + + // Min + minDB := filepath.Join(home, minHistoryPath) + minResults, err := b.ingestMin(minDB) + if err == nil { + results = append(results, minResults...) + } + + return results, nil +} + +func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) { + if !fileExists(path) { + return nil, nil + } + + // Copy to avoid lock issues (Chromium uses WAL) + tmp := path + ".knox_tmp" + if err := copyFile(path, tmp); err != nil { + return nil, fmt.Errorf("copy chromium db: %w", err) + } + defer os.Remove(tmp) + + db, err := sql.Open("sqlite", tmp) + if err != nil { + return nil, fmt.Errorf("open chromium copy: %w", err) + } + defer db.Close() + + rows, err := db.Query( + `SELECT url, COALESCE(title,''), last_visit_time, visit_count + FROM urls + WHERE last_visit_time > 0 + ORDER BY last_visit_time DESC + LIMIT 1000`, + ) + if err != nil { + // Table might not exist or schema mismatch + return nil, fmt.Errorf("query chromium urls: %w", err) + } + defer rows.Close() + + var results []*IngestResult + for rows.Next() { + var url, title string + var visitTimeMicro int64 + var visitCount int + if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil { + continue + } + + // Chromium webkit time: microseconds since 1601-01-01 UTC + visitTime := webkitToTime(visitTimeMicro) + + if result := b.makeResult(url, title, visitTime, visitCount); result != nil { + results = append(results, result) + } + } + return results, nil +} + +func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) { + if !fileExists(path) { + return nil, nil + } + + tmp := path + ".knox_tmp" + if err := copyFile(path, tmp); err != nil { + return nil, fmt.Errorf("copy min db: %w", err) + } + defer os.Remove(tmp) + + db, err := sql.Open("sqlite", tmp) + if err != nil { + return nil, fmt.Errorf("open min copy: %w", err) + } + defer db.Close() + + rows, err := db.Query( + `SELECT url, COALESCE(title,''), timestamp + FROM history + ORDER BY timestamp DESC + LIMIT 1000`, + ) + if err != nil { + return nil, fmt.Errorf("query min history: %w", err) + } + defer rows.Close() + + var results []*IngestResult + for rows.Next() { + var url, title string + var ts int64 + if err := rows.Scan(&url, &title, &ts); err != nil { + continue + } + + visitTime := time.Unix(ts, 0) + if result := b.makeResult(url, title, visitTime, 1); result != nil { + results = append(results, result) + } + } + return results, nil +} + +func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult { + if url == "" { + return nil + } + + // Skip chrome://, about://, file://, devtools + if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") || + strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") || + strings.HasPrefix(url, "chrome-extension://") { + return nil + } + + if noiseURL(url) { + return nil + } + + displayTitle := title + if displayTitle == "" { + displayTitle = extractDomain(url) + } + + fp := Fingerprint([]byte(url)) + + confidence := 0.6 + if visitCount > 5 { + confidence = 0.9 + } else if visitCount > 1 { + confidence = 0.7 + } + + return &IngestResult{ + Fingerprint: fp, + SourceID: b.SourceID(), + SourcePath: url, + ContentType: "url", + Title: displayTitle, + Summary: url, + CreatedAt: visitTime.Format(time.RFC3339), + Confidence: confidence, + IngesterVersion: b.Version, + Provenance: map[string]any{ + "url": url, + "visit_count": visitCount, + "visited_at": visitTime.Format(time.RFC3339), + }, + } +} + +// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time +func webkitToTime(micros int64) time.Time { + // WebKit epoch: 1601-01-01 UTC + // Unix epoch: 1970-01-01 UTC + // Difference: 11644473600 seconds + secs := micros / 1_000_000 + if secs < 11644473600 { + return time.Time{} + } + return time.Unix(secs-11644473600, 0) +} + +func extractDomain(url string) string { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + parts := strings.Split(url, "/") + if len(parts) > 0 { + return parts[0] + } + return url +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func copyFile(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, data, 0644) +} diff --git a/internal/ingest/git.go b/internal/ingest/git.go new file mode 100644 index 0000000..e9eb5a0 --- /dev/null +++ b/internal/ingest/git.go @@ -0,0 +1,269 @@ +package ingest + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +// GitIngester tracks project status for local git repositories. +// +// For each repo under one or more scan roots (default ~/src) it records the +// current branch, whether the working tree is clean, ahead/behind counts vs +// the upstream, the HEAD hash, and the last commit time + subject. The entry's +// CreatedAt is the last commit time, so staleness/dedup works naturally and +// an entry updates when new commits land. +// +// When Recursive is set, repos are discovered at any depth under a root +// (following symlinked directories), not just direct children — this picks up +// nested repos like ~/assistant/kitmaker or ~/Arduino/*/... Repos already have +// a stable identity via their real (symlink-resolved) path, so overlapping +// roots across symlinks do not duplicate entries. +type GitIngester struct { + Version string + Root string // single scan root; overridden by Roots (empty => $HOME/src) + Roots []string // multi-scan roots; overrides Root when non-empty + Recursive bool // recurse into subdirs (and follow dir symlinks) to find nested .git +} + +func NewGitIngester() *GitIngester { + return &GitIngester{Version: "git/v1"} +} + +func (g *GitIngester) SourceID() string { return "git" } + +// scanRoots returns the effective list of scan roots. +func (g *GitIngester) scanRoots() []string { + if len(g.Roots) > 0 { + return g.Roots + } + if g.Root != "" { + return []string{g.Root} + } + home, err := os.UserHomeDir() + if err != nil { + return nil + } + return []string{filepath.Join(home, "src")} +} + +func (g *GitIngester) IngestAll() ([]*IngestResult, error) { + // Dedup by real path so overlapping roots / symlinked roots don't double-add. + seen := make(map[string]struct{}) + var results []*IngestResult + for _, root := range g.scanRoots() { + for _, repoPath := range collectGitRepos(root, g.Recursive) { + real, err := filepath.EvalSymlinks(repoPath) + if err != nil { + real = repoPath + } + if _, dup := seen[real]; dup { + continue + } + seen[real] = struct{}{} + if r := g.repoToResult(real); r != nil { + results = append(results, r) + } + } + } + return results, nil +} + +func isGitRepo(path string) bool { + info, err := os.Stat(filepath.Join(path, ".git")) + return err == nil && info.IsDir() +} + +// collectGitRepos returns paths of git repos under root. In non-recursive mode +// only direct children are considered (root itself is not counted), matching +// the original behavior. In recursive mode the root is walked to any depth, +// descending into symlinked directories, up to and including the root if it is +// itself a git repo. Descending stops once a git repo is found, so nested +// repos (e.g. a vendored copy inside another repo) are treated as leaves. +func collectGitRepos(root string, recursive bool) []string { + visited := make(map[string]struct{}) + var out []string + + var walk func(dir string) + walk = func(dir string) { + real, err := filepath.EvalSymlinks(dir) + if err != nil { + return + } + if _, ok := visited[real]; ok { + return + } + visited[real] = struct{}{} + + if isGitRepo(real) { + out = append(out, real) + return + } + + entries, err := os.ReadDir(real) + if err != nil { + return + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".") { + continue + } + child := filepath.Join(real, e.Name()) + if recursive { + walk(child) + continue + } + if e.IsDir() && isGitRepo(child) { + out = append(out, child) + } + } + } + + walk(root) + return out +} + +type GitStatus struct { + Branch string + Head string + Dirty int + Ahead int + Behind int + LastCommit string + Subject string +} + +func (g *GitIngester) repoToResult(repoPath string) *IngestResult { + status := gitStatus(repoPath) + name := filepath.Base(repoPath) + fp := Fingerprint([]byte("git:repo:" + repoPath)) + + clean := "clean" + if status.Dirty > 0 { + clean = fmt.Sprintf("%d change(s)", status.Dirty) + } + + summary := fmt.Sprintf("%s @ %s — %s", name, status.Branch, clean) + if status.Ahead > 0 || status.Behind > 0 { + summary += fmt.Sprintf(", ahead %d / behind %d", status.Ahead, status.Behind) + } + if status.Subject != "" { + summary += " — “" + truncate(status.Subject, 80) + "”" + } + + createdAt := status.LastCommit + if createdAt == "" { + createdAt = gitStatusTime(status) + } + + return &IngestResult{ + Fingerprint: fp, + SourceID: g.SourceID(), + SourcePath: repoPath, + Project: name, + ContentType: "git-status", + Title: name, + Summary: summary, + CreatedAt: createdAt, + Confidence: 0.9, + IngesterVersion: g.Version, + Provenance: map[string]any{ + "type": "git_status", + "path": repoPath, + "branch": status.Branch, + "head": status.Head, + "dirty": status.Dirty, + "ahead": status.Ahead, + "behind": status.Behind, + "last_commit": status.LastCommit, + "subject": status.Subject, + }, + } +} + +func gitStatusTime(s GitStatus) string { + if s.LastCommit != "" { + return s.LastCommit + } + return time.Now().UTC().Format(time.RFC3339) +} + +func gitStatus(repoPath string) GitStatus { + out, err := gitOut(repoPath, "status", "--porcelain") + if err != nil { + return GitStatus{} + } + dirty := 0 + if out != "" { + dirty = len(strings.Split(strings.TrimRight(out, "\n"), "\n")) + } + + branch, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD") + head, _ := gitOut(repoPath, "rev-parse", "--short", "HEAD") + lastCommit, _ := gitOut(repoPath, "log", "-1", "--format=%cI") + subject, _ := gitOut(repoPath, "log", "-1", "--format=%s") + + ahead, behind := 0, 0 + // ahead/behind only meaningful if there's an upstream + if upOut, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD@{upstream}"); upOut != "" { + if rev, err := gitOut(repoPath, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err == nil { + fields := strings.Fields(rev) + if len(fields) == 2 { + var a, b int + if _, e1 := fmt.Sscanf(fields[0], "%d", &a); e1 == nil { + ahead = a + } + if _, e2 := fmt.Sscanf(fields[1], "%d", &b); e2 == nil { + behind = b + } + } + } + } + + return GitStatus{ + Branch: strings.TrimSpace(branch), + Head: strings.TrimSpace(head), + Dirty: dirty, + Ahead: ahead, + Behind: behind, + LastCommit: strings.TrimSpace(lastCommit), + Subject: strings.TrimSpace(subject), + } +} + +// sanitizeGitField strips ANSI codes and control bytes from git output +// (colour/unicode modes can inject escape sequences). +var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +func sanitizeGitField(s string) string { + s = ansiRe.ReplaceAllString(s, "") + s = strings.ReplaceAll(s, "\x1b", "") + return strings.TrimSpace(s) +} + +// gitOut runs git in repoPath and returns trimmed stdout (or "" on error). +func gitOut(repoPath string, args ...string) (string, error) { + cmd := exec.Command("git", append([]string{"-C", repoPath}, args...)...) + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1") + out, err := cmd.Output() + if err != nil { + return "", err + } + return sanitizeGitField(string(out)), nil +} + +func (g *GitIngester) Ingest(_ string) (*IngestResult, error) { + return nil, fmt.Errorf("use IngestAll() for git") +} + +var _ Ingester = (*GitIngester)(nil) + +// SortResults orders results deterministically by project name (helper for CLI output). +func SortResults(res []*IngestResult) { + sort.Slice(res, func(i, j int) bool { return res[i].Project < res[j].Project }) +} diff --git a/internal/ingest/gitea.go b/internal/ingest/gitea.go new file mode 100644 index 0000000..050e113 --- /dev/null +++ b/internal/ingest/gitea.go @@ -0,0 +1,245 @@ +package ingest + +import ( + "encoding/json" + "fmt" + "os/exec" + "time" +) + +type GiteaIngester struct { + Version string +} + +func NewGiteaIngester() *GiteaIngester { + return &GiteaIngester{Version: "gitea-tea/v1"} +} + +func (g *GiteaIngester) SourceID() string { return "gitea" } + +func (g *GiteaIngester) IngestAll() ([]*IngestResult, error) { + var results []*IngestResult + + repos, err := g.fetchRepos() + if err == nil { + for _, r := range repos { + if result := g.repoToResult(r); result != nil { + results = append(results, result) + } + } + } + + issues, err := g.fetchIssues() + if err == nil { + for _, i := range issues { + if result := g.issueToResult(i); result != nil { + results = append(results, result) + } + } + } + + pulls, err := g.fetchPulls() + if err == nil { + for _, p := range pulls { + if result := g.pullToResult(p); result != nil { + results = append(results, result) + } + } + } + + return results, nil +} + +type teaRepo struct { + Owner string `json:"owner"` + Name string `json:"name"` + Description string `json:"description"` + Updated string `json:"updated"` +} + +type teaIssue struct { + Index string `json:"index"` + Title string `json:"title"` + State string `json:"state"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Labels string `json:"labels"` + Author string `json:"author"` + Updated string `json:"updated"` +} + +type teaPull struct { + Index string `json:"index"` + Title string `json:"title"` + State string `json:"state"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Updated string `json:"updated"` +} + +func (g *GiteaIngester) teaJSON(args ...string) ([][]byte, error) { + cmd := exec.Command("tea", append(args, "--output", "json")...) + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("tea %s: %w", args[0], err) + } + + var raw json.RawMessage + if err := json.Unmarshal(output, &raw); err != nil { + return nil, err + } + + // Handle both array and object responses + var arr []json.RawMessage + if err := json.Unmarshal(raw, &arr); err == nil { + bytes := make([][]byte, len(arr)) + for i, a := range arr { + bytes[i] = a + } + return bytes, nil + } + + // Single object — wrap in array + return [][]byte{raw}, nil +} + +func (g *GiteaIngester) fetchRepos() ([]teaRepo, error) { + items, err := g.teaJSON("repos", "list", "--limit", "100", "--fields", "owner,name,description,updated") + if err != nil { + return nil, err + } + var repos []teaRepo + for _, item := range items { + var r teaRepo + if err := json.Unmarshal(item, &r); err == nil { + repos = append(repos, r) + } + } + return repos, nil +} + +func (g *GiteaIngester) fetchIssues() ([]teaIssue, error) { + items, err := g.teaJSON("issues", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,author,milestone,labels,owner,repo,updated") + if err != nil { + return nil, err + } + var issues []teaIssue + for _, item := range items { + var i teaIssue + if err := json.Unmarshal(item, &i); err == nil { + issues = append(issues, i) + } + } + return issues, nil +} + +func (g *GiteaIngester) fetchPulls() ([]teaPull, error) { + items, err := g.teaJSON("pulls", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,owner,repo,updated") + if err != nil { + return nil, err + } + var pulls []teaPull + for _, item := range items { + var p teaPull + if err := json.Unmarshal(item, &p); err == nil { + pulls = append(pulls, p) + } + } + return pulls, nil +} + +// signalTime returns the item's last-activity time, falling back to now. +func signalTime(updated string) string { + if updated != "" { + if _, err := time.Parse(time.RFC3339, updated); err == nil { + return updated + } + } + return time.Now().UTC().Format(time.RFC3339) +} + +func (g *GiteaIngester) repoToResult(r teaRepo) *IngestResult { + fullName := r.Owner + "/" + r.Name + fp := Fingerprint([]byte("gitea:repo:" + fullName)) + + return &IngestResult{ + Fingerprint: fp, + SourceID: g.SourceID(), + SourcePath: fullName, + Project: r.Name, + ContentType: "repo", + Title: fullName, + Summary: r.Description, + Confidence: 0.9, + IngesterVersion: g.Version, + CreatedAt: signalTime(r.Updated), + Provenance: map[string]any{ + "type": "repository", + "owner": r.Owner, + "name": r.Name, + "description": r.Description, + }, + } +} + +func (g *GiteaIngester) issueToResult(i teaIssue) *IngestResult { + title := fmt.Sprintf("#%s %s", i.Index, i.Title) + fp := Fingerprint([]byte(fmt.Sprintf("gitea:issue:%s/%s:%s", i.Owner, i.Repo, i.Index))) + + summary := fmt.Sprintf("[%s/%s#%s] %s", i.Owner, i.Repo, i.Index, i.Title) + if i.Labels != "" { + summary += " [" + i.Labels + "]" + } + + return &IngestResult{ + Fingerprint: fp, + SourceID: g.SourceID(), + SourcePath: fmt.Sprintf("%s/%s#%s", i.Owner, i.Repo, i.Index), + Project: i.Repo, + ContentType: "issue", + Title: title, + Summary: summary, + Confidence: 0.85, + IngesterVersion: g.Version, + CreatedAt: signalTime(i.Updated), + Provenance: map[string]any{ + "type": "issue", + "owner": i.Owner, + "repo": i.Repo, + "index": i.Index, + "state": i.State, + "labels": i.Labels, + }, + } +} + +func (g *GiteaIngester) pullToResult(p teaPull) *IngestResult { + title := fmt.Sprintf("!%s %s", p.Index, p.Title) + fp := Fingerprint([]byte(fmt.Sprintf("gitea:pull:%s/%s:%s", p.Owner, p.Repo, p.Index))) + + return &IngestResult{ + Fingerprint: fp, + SourceID: g.SourceID(), + SourcePath: fmt.Sprintf("%s/%s!%s", p.Owner, p.Repo, p.Index), + Project: p.Repo, + ContentType: "pull", + Title: title, + Summary: fmt.Sprintf("[PR %s/%s] %s", p.Owner, p.Repo, p.Title), + Confidence: 0.85, + IngesterVersion: g.Version, + CreatedAt: signalTime(p.Updated), + Provenance: map[string]any{ + "type": "pull_request", + "owner": p.Owner, + "repo": p.Repo, + "index": p.Index, + "state": p.State, + }, + } +} + +func (g *GiteaIngester) Ingest(_ string) (*IngestResult, error) { + return nil, fmt.Errorf("use IngestAll() for gitea") +} + +var _ Ingester = (*GiteaIngester)(nil) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go new file mode 100644 index 0000000..2f2324b --- /dev/null +++ b/internal/ingest/ingest.go @@ -0,0 +1,57 @@ +package ingest + +import ( + "crypto/sha256" + "encoding/json" + "fmt" +) + +type IngestResult struct { + Fingerprint string + SourceID string + SourcePath string + Project string + ContentType string + Title string + Summary string + CreatedAt string + LineStart int + LineEnd int + Confidence float64 + IngesterVersion string + Provenance map[string]any +} + +type Ingester interface { + SourceID() string + Ingest(path string) (*IngestResult, error) +} + +func Fingerprint(data []byte) string { + h := sha256.Sum256(data) + return fmt.Sprintf("%x", h[:16]) +} + +func FingerprintWithMeta(data []byte, meta map[string]string) string { + h := sha256.New() + h.Write(data) + enc := json.NewEncoder(h) + _ = enc.Encode(meta) + return fmt.Sprintf("%x", h.Sum(nil)[:16]) +} + +func ProvenanceJSON(m map[string]any) string { + b, err := json.Marshal(m) + if err != nil { + return "{}" + } + return string(b) +} + +func truncate(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) + "..." +} diff --git a/internal/ingest/log.go b/internal/ingest/log.go new file mode 100644 index 0000000..c564550 --- /dev/null +++ b/internal/ingest/log.go @@ -0,0 +1,138 @@ +package ingest + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +var _ Ingester = (*LogIngester)(nil) + +type LogIngester struct { + Version string +} + +func NewLogIngester() *LogIngester { + return &LogIngester{Version: "log-parser/v1"} +} + +func (l *LogIngester) SourceID() string { return "opencode-log" } + +var ( + sessionIDRE = regexp.MustCompile(`session[=_ ]?([a-zA-Z0-9_-]+)`) + projectIDRE = regexp.MustCompile(`project[=_ ]?([a-zA-Z0-9_.-]+)`) + modelRE = regexp.MustCompile(`model[=_ ]?([a-zA-Z0-9_.-]+)`) + agentRE = regexp.MustCompile(`agent[=_ ]?([a-zA-Z0-9_.-]+)`) + errorRE = regexp.MustCompile(`(?i)(error|fail|exception|panic|timeout)`) +) + +func (l *LogIngester) Ingest(path string) (*IngestResult, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open log: %w", err) + } + defer f.Close() + + var ( + sessionIDs []string + models []string + agents []string + errors []string + totalLines int + firstError int + ) + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + totalLines++ + + if m := sessionIDRE.FindStringSubmatch(line); len(m) > 1 { + sessionIDs = append(sessionIDs, m[1]) + } + if m := modelRE.FindStringSubmatch(line); len(m) > 1 { + models = append(models, m[1]) + } + if m := agentRE.FindStringSubmatch(line); len(m) > 1 { + agents = append(agents, m[1]) + } + if errorRE.MatchString(line) { + if len(errors) == 0 { + firstError = totalLines + } + errors = append(errors, truncate(line, 120)) + } + } + + base := filepath.Base(path) + fp := Fingerprint([]byte(path)) + title := fmt.Sprintf("Log %s (%d lines)", base, totalLines) + createdAt := "" + if fi, err := os.Stat(path); err == nil { + createdAt = fi.ModTime().UTC().Format(time.RFC3339) + } + + var summaryParts []string + if len(sessionIDs) > 0 { + summaryParts = append(summaryParts, fmt.Sprintf("sessions: %s", uniqueJoin(sessionIDs, 5))) + } + if len(models) > 0 { + summaryParts = append(summaryParts, fmt.Sprintf("models: %s", uniqueJoin(models, 3))) + } + if len(errors) > 0 { + summaryParts = append(summaryParts, fmt.Sprintf("%d errors (line %d)", len(errors), firstError)) + } + summary := strings.Join(summaryParts, " | ") + if summary == "" { + summary = fmt.Sprintf("%d lines", totalLines) + } + + confidence := 0.6 + if len(errors) > 0 { + confidence = 0.8 + } + + return &IngestResult{ + Fingerprint: fp, + SourceID: l.SourceID(), + SourcePath: path, + ContentType: ".log", + Title: title, + Summary: summary, + CreatedAt: createdAt, + Confidence: confidence, + IngesterVersion: l.Version, + LineEnd: totalLines, + Provenance: map[string]any{ + "session_ids": uniqueSlice(sessionIDs), + "models": uniqueSlice(models), + "agents": uniqueSlice(agents), + "error_count": len(errors), + "file_size": fmt.Sprintf("%d lines", totalLines), + }, + }, nil +} + +func uniqueJoin(items []string, max int) string { + uniq := uniqueSlice(items) + if len(uniq) > max { + uniq = uniq[:max] + } + return strings.Join(uniq, ", ") +} + +func uniqueSlice(items []string) []string { + seen := make(map[string]bool) + var uniq []string + for _, s := range items { + if !seen[s] { + seen[s] = true + uniq = append(uniq, s) + } + } + return uniq +} diff --git a/internal/ingest/obsidian.go b/internal/ingest/obsidian.go new file mode 100644 index 0000000..99d6549 --- /dev/null +++ b/internal/ingest/obsidian.go @@ -0,0 +1,267 @@ +package ingest + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const obsidianConfigPath = ".config/obsidian/obsidian.json" + +type ObsidianIngester struct { + Version string + Vault string // optional override; empty = auto-detect +} + +func NewObsidianIngester(vault string) *ObsidianIngester { + return &ObsidianIngester{Version: "obsidian-note/v1", Vault: vault} +} + +func (o *ObsidianIngester) SourceID() string { return "obsidian" } + +func (o *ObsidianIngester) IngestAll() ([]*IngestResult, error) { + vault := o.Vault + if vault == "" { + var err error + vault, err = detectObsidianVault() + if err != nil { + return nil, fmt.Errorf("no vault path and auto-detect failed: %w", err) + } + } + + if !fileExists(vault) { + return nil, fmt.Errorf("vault path %s does not exist", vault) + } + + var results []*IngestResult + err := filepath.Walk(vault, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil // skip errors per-file + } + if info.IsDir() { + dir := filepath.Base(path) + if strings.HasPrefix(dir, ".") || dir == "node_modules" { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".md") { + return nil + } + + result, err := o.ingestNote(path, vault) + if err != nil { + return nil + } + if result != nil { + results = append(results, result) + } + return nil + }) + + return results, err +} + +func (o *ObsidianIngester) ingestNote(path, vault string) (*IngestResult, error) { + data, err := os.ReadFile(path) + if err != nil || len(data) < 10 { + return nil, nil + } + + content := string(data) + title := extractObsidianTitle(content, path, vault) + tags := extractObsidianTags(content) + created, modified := extractObsidianTimestamps(content, path) + if created == "" { + if fi, err := os.Stat(path); err == nil { + created = fi.ModTime().UTC().Format(time.RFC3339) + } + } + if modified == "" && created != "" { + modified = created + } + + // Fingerprint by content (so edits create updates) + fp := FingerprintWithMeta(data, map[string]string{ + "source": "obsidian", + "path": path, + }) + + relPath, _ := filepath.Rel(vault, path) + summary := extractBodyPreview(content, 200) + + return &IngestResult{ + Fingerprint: fp, + SourceID: o.SourceID(), + SourcePath: relPath, + Project: filepath.Base(vault), + ContentType: ".md", + Title: title, + Summary: summary, + CreatedAt: created, + LineStart: 0, + LineEnd: len(strings.Split(content, "\n")), + Confidence: 0.85, + IngesterVersion: o.Version, + Provenance: map[string]any{ + "path": relPath, + "tags": tags, + "modified_at": modified, + "note_title": title, + }, + }, nil +} + +func DetectObsidianVault() (string, error) { return detectObsidianVault() } +func detectObsidianVault() (string, error) { + home, _ := os.UserHomeDir() + cfgPath := filepath.Join(home, obsidianConfigPath) + if !fileExists(cfgPath) { + // Fallback: check common vault locations + common := []string{ + filepath.Join(home, "notes"), + filepath.Join(home, "Documents", "notes"), + filepath.Join(home, "Obsidian"), + filepath.Join(home, "vault"), + } + for _, p := range common { + if fileExists(filepath.Join(p, ".obsidian")) { + return p, nil + } + } + return "", fmt.Errorf("no obsidian config found at %s and no common vault detected", cfgPath) + } + + data, err := os.ReadFile(cfgPath) + if err != nil { + return "", err + } + + var cfg struct { + Vaults map[string]struct { + Path string `json:"path"` + } `json:"vaults"` + } + if err := json.Unmarshal(data, &cfg); err != nil { + return "", fmt.Errorf("parse obsidian config: %w", err) + } + + for _, v := range cfg.Vaults { + if v.Path != "" { + return v.Path, nil + } + } + return "", fmt.Errorf("no vault path in obsidian config") +} + +func extractObsidianTitle(content, path, vault string) string { + // Try frontmatter title first + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "title:") { + t := strings.TrimSpace(trimmed[6:]) + t = strings.Trim(t, "\"'") + if t != "" { + return t + } + } + if strings.HasPrefix(trimmed, "aliases:") { + break + } + if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 { + break + } + } + + // Fallback: first H1 + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "# ") { + return strings.TrimSpace(trimmed[2:]) + } + } + + // Fallback: filename + rel, _ := filepath.Rel(vault, path) + return strings.TrimSuffix(rel, ".md") +} + +func extractObsidianTags(content string) []string { + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "tags:") { + raw := strings.TrimSpace(trimmed[5:]) + raw = strings.Trim(raw, "\"[] ") + var tags []string + for _, t := range strings.Split(raw, ",") { + t = strings.TrimSpace(t) + t = strings.Trim(t, "\" ") + if t != "" { + tags = append(tags, t) + } + } + return tags + } + if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 { + break + } + } + return nil +} + +func extractObsidianTimestamps(content, path string) (created, modified string) { + // Try frontmatter dates + inFrontmatter := false + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "---" { + inFrontmatter = !inFrontmatter + continue + } + if !inFrontmatter { + break + } + if strings.HasPrefix(trimmed, "created:") || strings.HasPrefix(trimmed, "date:") { + created = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:]) + created = strings.Trim(created, "\"' ") + } + if strings.HasPrefix(trimmed, "modified:") || strings.HasPrefix(trimmed, "updated:") { + modified = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:]) + modified = strings.Trim(modified, "\"' ") + } + } + return created, modified +} + +func extractBodyPreview(content string, maxLen int) string { + inFrontmatter := false + var body strings.Builder + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "---" { + if inFrontmatter { + inFrontmatter = false + continue + } + inFrontmatter = true + continue + } + if inFrontmatter { + continue + } + if trimmed != "" && body.Len() < maxLen { + if body.Len() > 0 { + body.WriteString(" ") + } + body.WriteString(truncate(trimmed, maxLen-body.Len())) + } + } + result := body.String() + if len(result) > maxLen { + return result[:maxLen] + "..." + } + return result +} diff --git a/internal/ingest/obsidian_file.go b/internal/ingest/obsidian_file.go new file mode 100644 index 0000000..e64c201 --- /dev/null +++ b/internal/ingest/obsidian_file.go @@ -0,0 +1,69 @@ +package ingest + +import ( + "os" + "path/filepath" + "strings" + "time" +) + +var _ Ingester = (*ObsidianFileIngester)(nil) + +type ObsidianFileIngester struct { + Version string + Vault string +} + +func NewObsidianFileIngester(vault string) *ObsidianFileIngester { + return &ObsidianFileIngester{Version: "obsidian-file/v1", Vault: vault} +} + +func (o *ObsidianFileIngester) SourceID() string { return "obsidian" } + +func (o *ObsidianFileIngester) Ingest(path string) (*IngestResult, error) { + if !strings.HasSuffix(path, ".md") { + return nil, nil + } + + data, err := os.ReadFile(path) + if err != nil || len(data) < 10 { + return nil, nil + } + + content := string(data) + title := extractObsidianTitle(content, path, o.Vault) + tags := extractObsidianTags(content) + created, _ := extractObsidianTimestamps(content, path) + if created == "" { + if fi, err := os.Stat(path); err == nil { + created = fi.ModTime().UTC().Format(time.RFC3339) + } + } + + fp := FingerprintWithMeta(data, map[string]string{ + "source": "obsidian", + "path": path, + }) + + relPath, _ := filepath.Rel(o.Vault, path) + summary := extractBodyPreview(content, 200) + + return &IngestResult{ + Fingerprint: fp, + SourceID: o.SourceID(), + SourcePath: relPath, + Project: filepath.Base(o.Vault), + ContentType: ".md", + Title: title, + Summary: summary, + CreatedAt: created, + LineEnd: len(strings.Split(content, "\n")), + Confidence: 0.85, + IngesterVersion: o.Version, + Provenance: map[string]any{ + "path": relPath, + "tags": tags, + "note_title": title, + }, + }, nil +} diff --git a/internal/ingest/session.go b/internal/ingest/session.go new file mode 100644 index 0000000..27f5b8e --- /dev/null +++ b/internal/ingest/session.go @@ -0,0 +1,157 @@ +package ingest + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +type jsonKind int + +const ( + jsonObject jsonKind = iota + jsonArray +) + +var _ Ingester = (*SessionDiffIngester)(nil) + +type SessionDiffIngester struct { + Version string +} + +func NewSessionDiffIngester() *SessionDiffIngester { + return &SessionDiffIngester{Version: "session-diff/v1"} +} + +func (s *SessionDiffIngester) SourceID() string { return "opencode-session" } + +func (s *SessionDiffIngester) Ingest(path string) (*IngestResult, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read session diff: %w", err) + } + if len(data) < 10 { + return nil, nil + } + + raw, kind := parseJSON(data) + if raw == nil { + return nil, nil + } + + sessionID := guessString(raw, "session_id", "id", "sessionId") + project := guessString(raw, "project", "project_id", "projectPath", "projectId") + title := guessString(raw, "title", "name", "label") + createdAt := guessString(raw, "created_at", "timestamp", "date", "started_at") + if createdAt == "" { + if fi, err := os.Stat(path); err == nil { + createdAt = fi.ModTime().UTC().Format(time.RFC3339) + } + } + + if sessionID == "" { + base := filepath.Base(path) + sessionID = strings.TrimSuffix(base, filepath.Ext(base)) + sessionID = strings.TrimPrefix(sessionID, "ses_") + } + if title == "" { + title = fmt.Sprintf("Session %s", sessionID[:min(8, len(sessionID))]) + } + + summary := extractSummary(raw, kind) + fp := FingerprintWithMeta(data, map[string]string{ + "session_id": sessionID, + "source": "session_diff", + }) + + confidence := 0.7 + if kind == jsonArray { + confidence = 0.4 + } + + return &IngestResult{ + Fingerprint: fp, + SourceID: s.SourceID(), + SourcePath: path, + Project: project, + ContentType: ".json", + Title: title, + Summary: summary, + CreatedAt: createdAt, + LineStart: 0, + LineEnd: 0, + Confidence: confidence, + IngesterVersion: s.Version, + Provenance: map[string]any{ + "session_id": sessionID, + "kind": kindName(kind), + "field_count": len(raw), + }, + }, nil +} + +func parseJSON(data []byte) (map[string]any, jsonKind) { + var obj map[string]any + if err := json.Unmarshal(data, &obj); err == nil { + return obj, jsonObject + } + var arr []any + if err := json.Unmarshal(data, &arr); err == nil { + m := make(map[string]any) + m["_count"] = float64(len(arr)) + m["_kind"] = "array" + if len(arr) > 0 { + if first, ok := arr[0].(map[string]any); ok { + for k, v := range first { + if str, ok := v.(string); ok && len(str) < 200 { + m["_sample_"+k] = str + break + } + } + } + } + return m, jsonArray + } + return nil, jsonObject +} + +func extractSummary(raw map[string]any, kind jsonKind) string { + if msg, ok := raw["message"].(string); ok && len(msg) > 0 { + return truncate(msg, 200) + } + if msgs, ok := raw["messages"].([]any); ok && len(msgs) > 0 { + parts := make([]string, 0, len(msgs)) + for _, m := range msgs { + if mm, ok := m.(map[string]any); ok { + if c, ok := mm["content"].(string); ok { + parts = append(parts, truncate(c, 100)) + } + } + } + return strings.Join(parts, " | ") + } + if count, ok := raw["_count"].(float64); ok && kind == jsonArray { + return fmt.Sprintf("Array with %.0f entries", count) + } + b, _ := json.Marshal(raw) + return truncate(string(b), 200) +} + +func guessString(m map[string]any, keys ...string) string { + for _, k := range keys { + if v, ok := m[k].(string); ok && v != "" { + return v + } + } + return "" +} + +func kindName(k jsonKind) string { + if k == jsonArray { + return "array" + } + return "object" +} diff --git a/internal/ingest/skills.go b/internal/ingest/skills.go new file mode 100644 index 0000000..f69b064 --- /dev/null +++ b/internal/ingest/skills.go @@ -0,0 +1,135 @@ +package ingest + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +var _ Ingester = (*SkillsIngester)(nil) + +type SkillsIngester struct { + Version string +} + +func NewSkillsIngester() *SkillsIngester { + return &SkillsIngester{Version: "skills-index/v1"} +} + +func (s *SkillsIngester) SourceID() string { return "skills-catalog" } + +func (s *SkillsIngester) Ingest(path string) (*IngestResult, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read skill: %w", err) + } + if len(data) < 20 { + return nil, nil + } + + content := string(data) + title := extractSkillName(content, path) + summary := extractSkillSummary(content) + tags := extractSkillTags(content) + projects := extractProjects(tags) + createdAt := extractCreatedAt(content) + + fp := FingerprintWithMeta(data, map[string]string{ + "source": "skills-catalog", + "title": title, + }) + + return &IngestResult{ + Fingerprint: fp, + SourceID: s.SourceID(), + SourcePath: path, + Project: projects, + ContentType: ".md", + Title: title, + Summary: summary, + CreatedAt: createdAt, + LineStart: 0, + LineEnd: len(strings.Split(content, "\n")), + Confidence: 0.9, + IngesterVersion: s.Version, + Provenance: map[string]any{ + "skill_name": title, + "tags": tags, + "line_count": len(strings.Split(content, "\n")), + }, + }, nil +} + +func extractSkillName(content, path string) string { + base := filepath.Base(filepath.Dir(path)) + if base != "" && base != "." && base != "/" { + return base + } + return strings.TrimSuffix(filepath.Base(path), ".md") +} + +func extractSkillSummary(content string) string { + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "description:") { + desc := strings.TrimSpace(trimmed[len("description:"):]) + if len(desc) > 200 { + desc = desc[:200] + "..." + } + return desc + } + } + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "# ") { + continue + } + if trimmed != "" && !strings.HasPrefix(trimmed, "---") { + return truncate(trimmed, 200) + } + } + return "" +} + +func extractSkillTags(content string) []string { + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "tags:") { + raw := strings.TrimSpace(trimmed[len("tags:"):]) + raw = strings.Trim(raw, "\"[]") + var tags []string + for _, t := range strings.Split(raw, ",") { + t = strings.TrimSpace(t) + t = strings.Trim(t, "\" ") + if t != "" { + tags = append(tags, t) + } + } + return tags + } + } + return nil +} + +func extractProjects(tags []string) string { + for _, t := range tags { + if strings.HasPrefix(t, "scope:project:") { + return strings.TrimPrefix(t, "scope:project:") + } + if strings.HasPrefix(t, "scope:machine:") { + return "[machine]" + } + } + return "" +} + +func extractCreatedAt(content string) string { + for _, line := range strings.Split(content, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "created_at:") || strings.HasPrefix(trimmed, "updated_at:") { + return strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:]) + } + } + return "" +} diff --git a/internal/server/layout.html b/internal/server/layout.html new file mode 100644 index 0000000..96cd620 --- /dev/null +++ b/internal/server/layout.html @@ -0,0 +1,299 @@ +{{define "head"}}<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="UTF-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>Knox{{if .title}} · {{.title}}{{end}} + + + + + +
+

Knox{{if .title}} / {{.title}}{{end}}

+ +
+{{end}} + +{{define "foot"}} + +{{end}} + +{{define "not_found"}} +{{template "head" .}} +
+
Page not found
+ Back to Dashboard +
+{{template "foot" .}} +{{end}} + +{{define "dashboard"}} +{{template "head" .}} +
+ {{if .golden}} +
+
Golden Thread
+
{{.golden.Title}}
+
{{.golden.Motivation}}
+
+ {{end}} + +
+ + +
+
+
Searching...
+ +
+
Loading stats...
+
+ +
+

Issues

+ {{if .gitea}} +
+ {{range .gitea}} + + {{trunc .Title 80}} + {{displayTime .}} + + {{end}} +
+ {{end}} +
+ +
+
+

Notes

+ {{if .obsidian}} +
+ {{range .obsidian}} + + {{trunc .Title 50}} + {{displayTime .}} + + {{end}} +
+ {{else}} +
No notes indexed.
+ {{end}} +
+ +
+

Sessions

+ {{if .session}} +
+ {{range .session}} + + {{trunc .Title 50}} + {{displayTime .}} + + {{end}} +
+ {{else}} +
No sessions tracked.
+ {{end}} +
+ +
+

Skills

+ {{if .skills}} +
+ {{range .skills}} + + {{trunc .Title 50}} + {{displayTime .}} + + {{end}} +
+ {{else}} +
No skills indexed.
+ {{end}} +
+
+ +
+

Feed

+
+ {{range .recent}} + + {{shortFP .Fingerprint}} + {{trunc .Title 60}} + {{.SourceID}} + {{displayTime .}} + + {{else}} +
No entries yet.
+ {{end}} +
+
+
+{{template "foot" .}} +{{end}} + +{{define "search_results"}} +{{if .results}} +
+
{{len .results}} results for "{{.query}}"
+ {{range .results}} + +
+ {{shortFP .Fingerprint}} + {{trunc .Title 70}} + {{.SourceID}} +
+ {{if .Summary}}
{{trunc .Summary 120}}
{{end}} + + {{end}} +
+{{else}} +

No results for "{{.query}}".

+{{end}} +{{end}} + +{{define "stats"}} +
+ {{range .stats}} +
+
{{.Label}}
+
{{.Value}}
+
+ {{end}} +
+{{end}} + +{{define "thread_list"}} +{{template "head" .}} +
+
+ All + Active + Stalled + Resolved +
+ {{range .threads}} + +
+ {{.Title}} + {{.Status}} + {{.Priority}} +
+ {{if .Motivation}}
{{.Motivation}}
{{end}} +
{{.EntryCount}} observations · {{humanTime .UpdatedAt}}
+ + {{else}} +
No threads found.
+ {{end}} +
+{{template "foot" .}} +{{end}} + +{{define "thread_detail"}} +{{template "head" .}} +
+
+
+

{{.thread.Title}}

+ {{if .isGolden}}GOLDEN{{end}} +
+
+ {{.thread.Status}} + {{.thread.Priority}} + Created {{humanTime .thread.CreatedAt}} +
+ {{if .thread.Motivation}}
Motivation: {{.thread.Motivation}}
{{end}} + {{if ne .thread.Provenance "{}"}}
Provenance: {{.thread.Provenance}}
{{end}} +
+ + {{if .provenance}} +
+

Provenance Graph

+
+ {{range .provenance}} +
└─ {{shortFP .Fingerprint}} [{{.Relation}}]
+ {{end}} +
+
+ {{end}} + + {{if .obs}} +
+

Observations ({{len .obs}})

+
+ {{range .obs}} +
+ #{{.ID}} + {{trunc .Title 50}} + {{.SourceID}} + {{if gt .Count 1}}×{{.Count}}{{end}} + {{humanTime .CollectedAt}} +
+ {{end}} +
+
+ {{end}} + + {{if .notes}} +
+

Notes

+
+ {{range .notes}} +
+
{{humanTime .CreatedAt}}
+
{{.Note}}
+
+ {{end}} +
+
+ {{end}} +
+{{template "foot" .}} +{{end}} + +{{define "entry_detail"}} +{{template "head" .}} +
+
+
+
Fingerprint
{{.entry.Fingerprint}}
+
Source
{{.entry.SourceID}}
+
Title
{{.entry.Title}}
+
Project
{{.entry.Project}}
+
Path
{{.entry.SourcePath}}
+
Type
{{.entry.ContentType}}
+
Signal Time
{{if .entry.CreatedAt}}{{humanTime .entry.CreatedAt}}{{else}}—{{end}}
+
Ingested
{{humanTime .entry.LastSeen}}
+
Ref Count
{{.entry.RefCount}}
+
Confidence
{{printf "%.1f" .entry.LastConfidence}}
+
+ {{if .entry.Summary}}
{{.entry.Summary}}
{{end}} +
+ + {{if .obs}} +
+

Observation History ({{len .obs}})

+
+ {{range .obs}} +
+ #{{.ID}} + {{humanTime .CollectedAt}} + {{if gt .Count 1}}×{{.Count}}{{end}} + {{.Trigger}} + {{.IngesterVersion}} + conf:{{printf "%.1f" .Confidence}} +
+ {{end}} +
+
+ {{end}} +
+{{template "foot" .}} +{{end}} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..cbaab96 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,283 @@ +package server + +import ( + "bytes" + _ "embed" + "fmt" + "html/template" + "log" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/david/knox/internal/db" +) + +type Server struct { + kdb *db.KnoxDB + mux *http.ServeMux + tmpl *template.Template +} + +func New(kdb *db.KnoxDB) *Server { + s := &Server{kdb: kdb, mux: http.NewServeMux()} + s.tmpl = template.Must(template.New("knox").Funcs(template.FuncMap{ + "shortFP": shortFP, + "trunc": truncStr, + "fpURL": fpURL, + "humanTime": humanTime, + "displayTime": displayTime, + }).Parse(layout)) + + s.mux.HandleFunc("GET /", s.dashboard) + s.mux.HandleFunc("GET /search", s.search) + s.mux.HandleFunc("GET /threads", s.threadList) + s.mux.HandleFunc("GET /threads/{id}", s.threadDetail) + s.mux.HandleFunc("GET /entries/{fp}", s.entryDetail) + s.mux.HandleFunc("GET /stats", s.stats) + return s +} + +func (s *Server) Serve(addr string) error { + log.Printf("[knox] web UI at http://%s", addr) + return http.ListenAndServe(addr, s.mux) +} + +// ─── Dashboard ─────────────────────────────────────────────── + +func (s *Server) dashboard(w http.ResponseWriter, r *http.Request) { + gitea, _ := s.kdb.RecentEntriesBySource("gitea", 3) + obsidian, _ := s.kdb.RecentEntriesBySource("obsidian", 3) + skills, _ := s.kdb.RecentEntriesBySource("skills-catalog", 2) + session, _ := s.kdb.RecentEntriesBySource("opencode-session", 3) + recent, _ := s.kdb.RecentEntries(8) + + goldenID, err := s.kdb.GoldenThreadID() + if err != nil { + http.Error(w, "failed to load golden thread", http.StatusInternalServerError) + return + } + var golden *db.Thread + if goldenID > 0 { + golden, _ = s.kdb.GetThread(goldenID) + } + s.render(w, "dashboard", map[string]any{ + "gitea": gitea, + "obsidian": obsidian, + "skills": skills, + "session": session, + "recent": recent, + "golden": golden, + }) +} + +// ─── Search ────────────────────────────────────────────────── + +func (s *Server) search(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if q == "" { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write([]byte("

Enter a query above.

")) + return + } + results, err := s.kdb.Search(q, 30) + if err != nil { + http.Error(w, "search failed", http.StatusInternalServerError) + return + } + s.render(w, "search_results", map[string]any{ + "query": q, + "results": results, + }) +} + +// ─── Thread List ───────────────────────────────────────────── + +func (s *Server) threadList(w http.ResponseWriter, r *http.Request) { + status := r.URL.Query().Get("status") + threads, err := s.kdb.ListThreads(status) + if err != nil { + http.Error(w, "failed to load threads", http.StatusInternalServerError) + return + } + s.render(w, "thread_list", map[string]any{ + "title": "Threads", + "threads": threads, + "status": status, + }) +} + +// ─── Thread Detail ─────────────────────────────────────────── + +func (s *Server) threadDetail(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil { + http.Error(w, "invalid thread id", http.StatusBadRequest) + return + } + t, err := s.kdb.GetThread(id) + if err != nil { + http.Error(w, "failed to load thread", http.StatusInternalServerError) + return + } + if t == nil { + s.render(w, "not_found", map[string]any{"title": "Thread Not Found"}, http.StatusNotFound) + return + } + obs, err := s.kdb.ThreadObservations(id) + if err != nil { + http.Error(w, "failed to load observations", http.StatusInternalServerError) + return + } + notes, err := s.kdb.ThreadNotes(id) + if err != nil { + http.Error(w, "failed to load notes", http.StatusInternalServerError) + return + } + provenance, err := s.kdb.ThreadProvenance(id) + if err != nil { + http.Error(w, "failed to load provenance", http.StatusInternalServerError) + return + } + goldenID, _ := s.kdb.GoldenThreadID() + + s.render(w, "thread_detail", map[string]any{ + "title": fmt.Sprintf("Thread #%d", id), + "thread": t, + "obs": obs, + "notes": notes, + "provenance": provenance, + "isGolden": goldenID == id, + }) +} + +// ─── Entry Detail ──────────────────────────────────────────── + +func (s *Server) entryDetail(w http.ResponseWriter, r *http.Request) { + fp := r.PathValue("fp") + entry, err := s.kdb.FindEntry(fp) + if err != nil { + http.Error(w, "failed to load entry", http.StatusInternalServerError) + return + } + if entry == nil { + // Try fingerprint prefix match (the UI displays 8-char prefixes) + entries, err := s.kdb.FindEntryByPrefix(fp, 2) + if err != nil { + http.Error(w, "failed to load entry", http.StatusInternalServerError) + return + } + if len(entries) == 1 { + entry = &entries[0] + } else { + s.render(w, "not_found", map[string]any{"title": "Entry Not Found"}, http.StatusNotFound) + return + } + } + obs, err := s.kdb.ObservationsByFingerprint(entry.Fingerprint, 20) + if err != nil { + http.Error(w, "failed to load observations", http.StatusInternalServerError) + return + } + s.render(w, "entry_detail", map[string]any{ + "title": "Entry " + shortFP(entry.Fingerprint), + "entry": entry, + "obs": obs, + }) +} + +// ─── Stats Fragment ────────────────────────────────────────── + +type statItem struct { + Label string + Value any +} + +func statItems(stats map[string]any) []statItem { + keys := make([]string, 0, len(stats)) + for k := range stats { + keys = append(keys, k) + } + sort.Strings(keys) + items := make([]statItem, 0, len(keys)) + for _, k := range keys { + items = append(items, statItem{ + Label: strings.ReplaceAll(k, "_", " "), + Value: stats[k], + }) + } + return items +} + +func (s *Server) stats(w http.ResponseWriter, r *http.Request) { + stats, err := s.kdb.Stats() + if err != nil { + http.Error(w, "failed to load stats", http.StatusInternalServerError) + return + } + s.render(w, "stats", map[string]any{"stats": statItems(stats)}) +} + +// ─── Render ────────────────────────────────────────────────── + +func (s *Server) render(w http.ResponseWriter, name string, data map[string]any, status ...int) { + code := http.StatusOK + if len(status) > 0 { + code = status[0] + } + var buf bytes.Buffer + if err := s.tmpl.ExecuteTemplate(&buf, name, data); err != nil { + log.Printf("[knox] template error: %v", err) + http.Error(w, "template error", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(code) + buf.WriteTo(w) +} + +func shortFP(fp string) string { + if len(fp) > 8 { + return fp[:8] + } + return fp +} + +// fpURL routes synthetic thread fingerprints to thread pages. +func fpURL(fp string) string { + if id, ok := strings.CutPrefix(fp, "thread:"); ok { + return "/threads/" + id + } + return "/entries/" + fp +} + +func truncStr(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) + "..." +} + +func displayTime(e db.Entry) string { + if e.CreatedAt != "" { + return humanTime(e.CreatedAt) + } + return humanTime(e.LastSeen) +} + +func humanTime(s string) string { + formats := []string{time.RFC3339, "2006-01-02 15:04:05"} + for _, f := range formats { + t, err := time.Parse(f, s) + if err == nil { + return t.Local().Format("Jan 2, 2006 · 15:04") + } + } + return s +} + +//go:embed layout.html +var layout string diff --git a/internal/watch/threader.go b/internal/watch/threader.go new file mode 100644 index 0000000..a0cd752 --- /dev/null +++ b/internal/watch/threader.go @@ -0,0 +1,311 @@ +package watch + +import ( + "encoding/json" + "fmt" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/index" +) + +// AutoThreader promotes high-intent topic clusters into knowledge threads. +// +// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the +// "intent bar" when it shows sustained, recently-observed activity from a +// non-passive work source (git/gitea/session/obsidian) plus enough distinct +// sources overall. Motivation and title are synthesized from the cluster's +// strongest members — they can be refined later, but the grouping and linkage +// are deterministic and cheap enough for the watch daemon's periodic tick. +type AutoThreader struct { + DB *db.KnoxDB + MinSources int // distinct sources required (incl. one work source) + MinEntries int // minimum cluster size + Recency time.Duration // observations must be seen within this window + DryRun bool +} + +// workSources are signal sources that imply active intent (vs passive browsing +// or an always-present skills catalog). +var workSources = map[string]bool{ + "git": true, + "gitea": true, + "opencode-session": true, + "obsidian": true, + "filesystem": true, +} + +// NewAutoThreader reads the tuning knobs from the environment. All have +// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off. +func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader { + t := &AutoThreader{ + DB: kdb, + MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2), + MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3), + Recency: time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour, + } + return t +} + +func envInt(key string, def int) int { + if v, err := strconv.Atoi(os.Getenv(key)); err == nil { + return v + } + return def +} + +// AutoThread clusters recent entries and creates/extends threads for clusters +// crossing the intent bar. Returns the number of threads created and the total +// observations linked. +func (t *AutoThreader) AutoThread() (created, linked int, err error) { + entries, err := t.DB.RecentEntries(2000) + if err != nil { + return 0, 0, err + } + if len(entries) == 0 { + return 0, 0, nil + } + + tfidf := index.BuildTFIDF(entries) + clusters := tfidf.Cluster(2, 30) + now := time.Now().UTC() + + for _, c := range clusters { + fps := fingerprints(c.Entries) + if len(fps) == 0 || !t.crossesBar(c, fps, now) { + continue + } + + // Fold into an existing active thread if one already covers these terms. + if existing := t.findExisting(c); existing > 0 { + if t.DryRun { + fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name) + continue + } + n, err := t.DB.AutoLinkThreadObservations(existing, fps) + if err != nil { + log.Printf("[knox] threader link err: %v", err) + continue + } + if n > 0 { + log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n) + } + linked += n + continue + } + + title := t.title(c) + motivation := t.motivation(c, fps) + tags := t.tags(c) + priority := t.priority(c) + prov := map[string]any{ + "trigger": "auto_thread", + "cluster": c.Name, + "score": c.Score, + "sources": sourceList(c.Entries), + } + provJSON, _ := json.Marshal(prov) + + if t.DryRun { + fmt.Printf("[threader] (dry-run) would create thread %q [%s]\n", title, priority) + fmt.Printf(" motivation: %s\n", motivation) + fmt.Printf(" tags: %s\n", tags) + fmt.Printf(" %d obs | sources: %v\n", len(fps), prov["sources"]) + continue + } + + id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON)) + if err != nil { + log.Printf("[knox] threader create err: %v", err) + continue + } + if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil { + linked += n + } + created++ + log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority) + } + return created, linked, nil +} + +// crossesBar decides whether a cluster reflects real intent. +func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool { + if len(c.Entries) < t.MinEntries { + return false + } + + bySource := make(map[string]bool) + hasWork := false + recent := false + for _, e := range c.Entries { + if e.SourceID != "" { + bySource[e.SourceID] = true + if workSources[e.SourceID] { + hasWork = true + } + } + if e.LastSeen != "" { + if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency { + recent = true + } + } + } + + if len(bySource) < t.MinSources { + return false + } + if !hasWork { + return false + } + return recent +} + +// findExisting looks for an already-active thread that plausibly covers this +// cluster, by matching a discriminative cluster keyword against thread titles. +func (t *AutoThreader) findExisting(c index.TopicCluster) int64 { + for _, kw := range c.Keywords { + id, err := t.DB.ActiveThreadByKeyword(kw) + if err != nil { + return 0 + } + if id == 0 { + continue + } + th, err := t.DB.GetThread(id) + if err != nil || th == nil { + continue + } + titleTokens := toSet(strings.ToLower(th.Title)) + for _, kw2 := range c.Keywords { + if titleTokens[kw2] { + return id + } + } + } + return 0 +} + +func (t *AutoThreader) title(c index.TopicCluster) string { + return c.Name +} + +func (t *AutoThreader) tags(c index.TopicCluster) string { + var keep []string + for _, kw := range c.Keywords { + if len(keep) >= 5 { + break + } + keep = append(keep, kw) + } + return strings.Join(keep, " ") +} + +func (t *AutoThreader) priority(c index.TopicCluster) string { + srcs := len(sourceList(c.Entries)) + if srcs >= 3 { + return "high" + } + return "medium" +} + +// motivation synthesizes "why" from the cluster's strongest members: project +// context, source spread, and a couple of representative titles/summaries. +func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string { + sources := sourceList(c.Entries) + + proj := dominantProject(c.Entries) + + var b strings.Builder + fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", ")) + if proj != "" { + fmt.Fprintf(&b, " Dominant project: %s.", proj) + } + fmt.Fprintf(&b, " Keywords: %s.", c.Name) + fmt.Fprintf(&b, " Representative observations:") + + shown := 0 + seen := make(map[string]bool) + for _, e := range c.Entries { + if shown >= 3 { + break + } + txt := strings.TrimSpace(e.Title) + if txt == "" { + txt = strings.TrimSpace(e.Summary) + } + if txt == "" || seen[txt] { + continue + } + seen[txt] = true + fmt.Fprintf(&b, " • %s", truncate(txt, 90)) + shown++ + } + return b.String() +} + +// --- small helpers --------------------------------------------------------- + +func fingerprints(entries []db.Entry) []string { + seen := make(map[string]bool) + var out []string + for _, e := range entries { + if e.Fingerprint == "" || seen[e.Fingerprint] { + continue + } + seen[e.Fingerprint] = true + out = append(out, e.Fingerprint) + } + return out +} + +func sourceList(entries []db.Entry) []string { + seen := make(map[string]bool) + var out []string + for _, e := range entries { + if e.SourceID == "" || seen[e.SourceID] { + continue + } + seen[e.SourceID] = true + out = append(out, e.SourceID) + } + return out +} + +func dominantProject(entries []db.Entry) string { + counts := make(map[string]int) + for _, e := range entries { + if e.Project != "" { + counts[e.Project]++ + } + } + best, n := "", 0 + for p, c := range counts { + if c > n { + best, n = p, c + } + } + if n*2 >= len(entries) { + return best + } + return "" +} + +func toSet(s string) map[string]bool { + set := make(map[string]bool) + for _, w := range strings.Fields(s) { + set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true + } + return set +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} diff --git a/internal/watch/watch.go b/internal/watch/watch.go new file mode 100644 index 0000000..0c79d91 --- /dev/null +++ b/internal/watch/watch.go @@ -0,0 +1,447 @@ +package watch + +import ( + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/david/knox/internal/db" + "github.com/david/knox/internal/ingest" + "github.com/fsnotify/fsnotify" +) + +const browserInterval = 5 * time.Minute + +type Watcher struct { + knoxDB *db.KnoxDB + dirs []string + vault string + debounce time.Duration + fileIngesters []ingest.Ingester +} + +func New(kdb *db.KnoxDB, dirs []string) *Watcher { + // Detect Obsidian vault + vault, _ := ingest.DetectObsidianVault() + + return &Watcher{ + knoxDB: kdb, + dirs: dirs, + vault: vault, + debounce: 2 * time.Second, + fileIngesters: []ingest.Ingester{ + ingest.NewSessionDiffIngester(), + ingest.NewLogIngester(), + ingest.NewSkillsIngester(), + }, + } +} + +func (w *Watcher) Start() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + defer watcher.Close() + + w.seed(watcher) + log.Printf("[knox] watching %d directories", len(w.dirs)) + + if w.vault != "" { + log.Printf("[knox] obsidian vault: %s", w.vault) + } + + debounceMap := make(map[string]time.Time) + browserTicker := time.NewTicker(browserInterval) + giteaTicker := time.NewTicker(10 * time.Minute) + gitTicker := time.NewTicker(10 * time.Minute) + threadTicker := time.NewTicker(10 * time.Minute) + browserRunning := false + giteaRunning := false + gitRunning := false + threadRunning := false + + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return nil + } + if !w.isRelevantEvent(event) { + continue + } + + now := time.Now() + if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce { + continue + } + debounceMap[event.Name] = now + + trigger := eventOpName(event.Op) + log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name)) + w.ingestFile(event.Name, trigger) + + case <-browserTicker.C: + if browserRunning { + continue + } + browserRunning = true + go func() { + defer func() { browserRunning = false }() + w.ingestBrowserHistory() + }() + case <-giteaTicker.C: + if giteaRunning { + continue + } + giteaRunning = true + go func() { + defer func() { giteaRunning = false }() + w.ingestGitea() + }() + case <-gitTicker.C: + if gitRunning { + continue + } + gitRunning = true + go func() { + defer func() { gitRunning = false }() + w.ingestGit() + }() + case <-threadTicker.C: + if threadRunning { + continue + } + threadRunning = true + go func() { + defer func() { threadRunning = false }() + w.autoThread() + }() + + case err, ok := <-watcher.Errors: + if !ok { + return nil + } + log.Printf("[knox] watch error: %v", err) + } + } +} + +func (w *Watcher) seed(watcher *fsnotify.Watcher) { + for _, dir := range w.dirs { + abs, _ := filepath.Abs(dir) + if err := watcher.Add(abs); err != nil { + log.Printf("[knox] cannot watch %s: %v", abs, err) + continue + } + log.Printf("[knox] watching %s", abs) + } + + // Watch Obsidian vault + if w.vault != "" { + if err := watcher.Add(w.vault); err != nil { + log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err) + } else { + log.Printf("[knox] watching %s (obsidian)", w.vault) + } + } + + // Seed existing files + for _, ing := range w.fileIngesters { + for _, dir := range w.dirs { + patterns := []string{ + filepath.Join(dir, "*"), + filepath.Join(dir, "*", "SKILL.md"), + } + for _, pattern := range patterns { + entries, _ := filepath.Glob(pattern) + for _, path := range entries { + if !MatchesIngester(path, ing.SourceID()) { + continue + } + w.ingestFileWith(path, ing, "seed") + } + } + } + } + + // Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian) + // and covers all depths — glob patterns would match dot-dirs and miss depth >2. + if w.vault != "" { + if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil { + for _, r := range notes { + w.recordResult(r, "seed") + } + } + } +} + +func (w *Watcher) ingestFile(path, trigger string) { + for _, ing := range w.fileIngesters { + if MatchesIngester(path, ing.SourceID()) { + w.ingestFileWith(path, ing, trigger) + return + } + } + // Obsidian: any .md file in the vault + if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") { + obs := ingest.NewObsidianFileIngester(w.vault) + w.ingestFileWith(path, obs, trigger) + } +} + +func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) { + result, err := ing.Ingest(path) + if err != nil { + log.Printf("[knox] ingest error %s: %v", path, err) + return + } + if result == nil { + return + } + + // Staleness check: skip when the signal time is known and unchanged. + // Empty CreatedAt must NOT skip — it would suppress everything. + if result.CreatedAt != "" { + if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt { + return + } + } + + w.recordResult(result, trigger) +} + +// recordResult persists one ingest result and fires side effects (session +// tracking, golden-thread linking happens inside RecordObservation). +func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) { + obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + LineStart: result.LineStart, + LineEnd: result.LineEnd, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: trigger, + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + log.Printf("[knox] db error: %v", err) + return + } + + if isNew { + log.Printf("[knox] new entry #%d: %s", obsID, result.Title) + } else { + log.Printf("[knox] updated entry #%d: %s", obsID, result.Title) + } + + if result.SourceID == "opencode-session" { + sessionID, _ := result.Provenance["session_id"].(string) + if sessionID != "" { + status := "active" + w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status) + } + } +} + +func (w *Watcher) ingestBrowserHistory() { + log.Printf("[knox] periodic browser history ingest...") + ing := ingest.NewBrowserHistoryIngester() + results, err := ing.IngestAll() + if err != nil { + log.Printf("[knox] browser ingest error: %v", err) + return + } + + var newCount, skipped int + for _, result := range results { + existing, _ := w.knoxDB.FindEntry(result.Fingerprint) + if existing != nil && existing.CreatedAt == result.CreatedAt { + skipped++ + continue + } + _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "browser_timer", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + continue + } + if isNew { + newCount++ + } + } + if newCount > 0 || skipped > 0 { + log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped) + } +} + +func (w *Watcher) ingestGitea() { + log.Printf("[knox] periodic gitea ingest...") + ing := ingest.NewGiteaIngester() + results, err := ing.IngestAll() + if err != nil { + log.Printf("[knox] gitea ingest error: %v", err) + return + } + + var newCount, skipped int + for _, result := range results { + existing, _ := w.knoxDB.FindEntry(result.Fingerprint) + if existing != nil && existing.CreatedAt == result.CreatedAt { + skipped++ + continue + } + _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "gitea_timer", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + continue + } + if isNew { + newCount++ + } + } + if newCount > 0 || skipped > 0 { + log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped) + } +} + +func (w *Watcher) ingestGit() { + log.Printf("[knox] periodic git status ingest...") + ing := ingest.NewGitIngester() + ing.Recursive = true + if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" { + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + ing.Roots = append(ing.Roots, part) + } + } + results, err := ing.IngestAll() + if err != nil { + log.Printf("[knox] git ingest error: %v", err) + return + } + + var newCount, skipped int + for _, result := range results { + existing, _ := w.knoxDB.FindEntry(result.Fingerprint) + if existing != nil && existing.CreatedAt == result.CreatedAt { + skipped++ + continue + } + _, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{ + Fingerprint: result.Fingerprint, + SourceID: result.SourceID, + SourcePath: result.SourcePath, + Project: result.Project, + ContentType: result.ContentType, + Title: result.Title, + Summary: result.Summary, + CreatedAt: result.CreatedAt, + Confidence: result.Confidence, + IngesterVersion: result.IngesterVersion, + Trigger: "git_timer", + Provenance: ingest.ProvenanceJSON(result.Provenance), + }) + if err != nil { + continue + } + if isNew { + newCount++ + } + } + if newCount > 0 || skipped > 0 { + log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped) + } +} + +// autoThread runs the heuristic auto-threader. Disabled entirely when +// KNOX_THREAD_DISABLE is non-empty. +func (w *Watcher) autoThread() { + if os.Getenv("KNOX_THREAD_DISABLE") != "" { + return + } + threader := NewAutoThreader(w.knoxDB) + created, linked, err := threader.AutoThread() + if err != nil { + log.Printf("[knox] auto-thread error: %v", err) + return + } + if created > 0 || linked > 0 { + log.Printf("[knox] auto-thread: %d created, %d linked", created, linked) + } +} + +func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool { + if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) { + return false + } + name := filepath.Base(event.Name) + // Opencode session diffs and logs + if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") { + return true + } + // Obsidian markdown files + if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") { + return true + } + return false +} + +func MatchesIngester(path, sourceID string) bool { + base := filepath.Base(path) + switch sourceID { + case "opencode-session": + return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json") + case "opencode-log": + return strings.HasSuffix(base, ".log") + case "skills-catalog": + return base == "SKILL.md" + } + return false +} + +func eventOpName(op fsnotify.Op) string { + switch { + case op.Has(fsnotify.Create): + return "inotify:CREATE" + case op.Has(fsnotify.Write): + return "inotify:WRITE" + case op.Has(fsnotify.Chmod): + return "inotify:CHMOD" + default: + return "inotify:UNKNOWN" + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..6eed36a --- /dev/null +++ b/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "log" + "os" + + mcpServer "github.com/mark3labs/mcp-go/server" + "github.com/spf13/cobra" + knoxcmd "github.com/david/knox/internal/cmd" + "github.com/david/knox/internal/db" + knoxserver "github.com/david/knox/internal/server" +) + +func newServeCmd(kdb *db.KnoxDB) *cobra.Command { + var port int + cmd := &cobra.Command{ + Use: "serve", + Short: "Start web UI for the knowledge index", + RunE: func(c *cobra.Command, args []string) error { + srv := knoxserver.New(kdb) + addr := fmt.Sprintf("localhost:%d", port) + log.Printf("[knox] web UI at http://%s", addr) + return srv.Serve(addr) + }, + } + cmd.Flags().IntVarP(&port, "port", "p", 8924, "HTTP port") + return cmd +} + +func main() { + dbPath := os.Getenv("KNOX_DB") + if dbPath == "" { + home, err := os.UserHomeDir() + if err != nil { + log.Fatalf("cannot get home dir: %v", err) + } + dbPath = home + "/.local/share/knox/index.db" + } + + kdb, err := db.Open(dbPath) + if err != nil { + log.Fatalf("cannot open db: %v", err) + } + defer kdb.Close() + + root := &cobra.Command{ + Use: "knox", + Short: "Knowledge index & session tracker for opencode", + Long: `Knox watches opencode session data, ingests knowledge fingerprints, +and maintains a searchable index. Use 'knox watch' for daemon mode.`, + } + + root.AddCommand(knoxcmd.NewWatchCmd(kdb)) + root.AddCommand(knoxcmd.NewStatusCmd(kdb)) + root.AddCommand(knoxcmd.NewQueryCmd(kdb)) + root.AddCommand(knoxcmd.NewRecentCmd(kdb)) + root.AddCommand(knoxcmd.NewReflectCmd(kdb)) + root.AddCommand(knoxcmd.NewIngestCmd(kdb)) + root.AddCommand(knoxcmd.NewThreadCmd(kdb)) + root.AddCommand(knoxcmd.NewBrowserCmd(kdb)) + root.AddCommand(knoxcmd.NewObsidianCmd(kdb)) + root.AddCommand(knoxcmd.NewTopicsCmd(kdb)) + root.AddCommand(knoxcmd.NewGiteaCmd(kdb)) + root.AddCommand(knoxcmd.NewGitCmd(kdb)) + root.AddCommand(newServeCmd(kdb)) + + // MCP server subcommand + mcpCmd := &cobra.Command{ + Use: "mcp", + Short: "Start MCP server for opencode integration", + RunE: func(c *cobra.Command, args []string) error { + s := knoxcmd.NewMCPServer(kdb) + log.Printf("[knox] starting MCP stdio server") + if err := mcpServer.ServeStdio(s); err != nil { + return err + } + return nil + }, + } + root.AddCommand(mcpCmd) + + if err := root.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +}