Initial commit: knox knowledge index

This commit is contained in:
2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
+115
View File
@@ -0,0 +1,115 @@
package cmd
import (
"fmt"
"log"
"strings"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
)
func NewBrowserCmd(kdb *db.KnoxDB) *cobra.Command {
cmd := &cobra.Command{
Use: "browser",
Short: "Ingest browser history from Chromium and Min into the knowledge index",
Long: `Reads browser history databases (Chromium and Min), fingerprints URLs,
and stores them as observations. Tracks what you've searched for and read.
Run periodically to capture browsing exhaust.`,
RunE: func(c *cobra.Command, args []string) error {
ing := ingest.NewBrowserHistoryIngester()
log.Printf("[knox] ingesting browser history...")
results, err := ing.IngestAll()
if err != nil {
return fmt.Errorf("browser ingest: %w", err)
}
if len(results) == 0 {
fmt.Println("No browser history found (Chromium/Min not detected or empty).")
return nil
}
var newCount, dupCount int
for _, result := range results {
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "browser_ingest",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
continue
}
if isNew {
newCount++
} else {
dupCount++
}
}
fmt.Printf("Browser history: %d new, %d already seen\n", newCount, dupCount)
// Show top domains
domains := make(map[string]int)
for _, r := range results {
domains[extractDomain(r.Summary)]++
}
fmt.Println("\nTop domains:")
for _, d := range topDomains(domains, 10) {
fmt.Printf(" %-40s %d\n", d.Name, d.Count)
}
return nil
},
}
return cmd
}
func topDomains(domains map[string]int, n int) []struct {
Name string
Count int
} {
var sorted []struct {
Name string
Count int
}
for name, count := range domains {
sorted = append(sorted, struct {
Name string
Count int
}{name, count})
}
// Simple bubble sort for small n
for i := 0; i < len(sorted)-1; i++ {
for j := 0; j < len(sorted)-1-i; j++ {
if sorted[j].Count < sorted[j+1].Count {
sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
}
}
}
if len(sorted) > n {
sorted = sorted[:n]
}
return sorted
}
func extractDomain(url string) string {
url = strings.TrimPrefix(url, "https://")
url = strings.TrimPrefix(url, "http://")
parts := strings.Split(url, "/")
if len(parts) > 0 {
return parts[0]
}
return ""
}
+71
View File
@@ -0,0 +1,71 @@
package cmd
import (
"fmt"
"log"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
"github.com/spf13/cobra"
)
func NewGitCmd(kdb *db.KnoxDB) *cobra.Command {
var roots []string
var recursive bool
cmd := &cobra.Command{
Args: cobra.NoArgs,
Use: "git",
Short: "Ingest local git repository status into the knowledge index",
Long: `Scans git repos under one or more root dirs (default ~/src) and records
each project's status — branch, clean/dirty, ahead/behind, HEAD, last commit.
With --recursive (default) repos are found at any depth, following symlinked
directories, so nested repos like ~/assistant/kitmaker are picked up. Use
'knox git' for a one-shot ingest, or rely on the watch daemon's git timer.`,
RunE: func(c *cobra.Command, args []string) error {
ing := &ingest.GitIngester{Roots: roots, Recursive: recursive}
log.Printf("[knox] ingesting git repo status...")
results, err := ing.IngestAll()
if err != nil {
return fmt.Errorf("git ingest: %w", err)
}
if len(results) == 0 {
fmt.Println("No git repos found.")
return nil
}
ingest.SortResults(results)
var newCount int
for _, result := range results {
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "git_ingest",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
continue
}
if isNew {
newCount++
}
fmt.Printf("%-24s %s\n", result.Project, result.Summary)
}
fmt.Printf("\nGit: %d repos (%d new)\n", len(results), newCount)
return nil
},
}
cmd.Flags().StringSliceVarP(&roots, "root", "r", nil, "Directories to scan for git repos (default ~/src; repeat or comma-separate for multiple roots)")
cmd.Flags().BoolVar(&recursive, "recursive", true, "Recurse into subdirectories (following symlinks) to find nested git repos")
return cmd
}
+71
View File
@@ -0,0 +1,71 @@
package cmd
import (
"fmt"
"log"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
)
func NewGiteaCmd(kdb *db.KnoxDB) *cobra.Command {
cmd := &cobra.Command{
Use: "gitea",
Short: "Ingest Gitea repositories, issues, and PRs into the knowledge index",
Long: `Uses the 'tea' CLI to fetch repositories, open issues, and open
pull requests from Gitea. Fingerprints each by ID for dedup.`,
RunE: func(c *cobra.Command, args []string) error {
ing := ingest.NewGiteaIngester()
log.Printf("[knox] ingesting Gitea data via tea...")
results, err := ing.IngestAll()
if err != nil {
return fmt.Errorf("gitea ingest: %w", err)
}
if len(results) == 0 {
fmt.Println("No Gitea data found.")
return nil
}
var repos, issues, pulls int
var newCount int
for _, result := range results {
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "gitea_ingest",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
continue
}
if isNew {
newCount++
}
switch result.ContentType {
case "repo":
repos++
case "issue":
issues++
case "pull":
pulls++
}
}
fmt.Printf("Gitea: %d repos, %d issues, %d PRs (%d new)\n", repos, issues, pulls, newCount)
return nil
},
}
return cmd
}
+120
View File
@@ -0,0 +1,120 @@
package cmd
import (
"fmt"
"log"
"path/filepath"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
"github.com/david/knox/internal/watch"
)
func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
var dirs []string
cmd := &cobra.Command{
Use: "ingest",
Short: "One-time full sweep of all sources into the knowledge index",
Run: func(cmd *cobra.Command, args []string) {
if len(dirs) == 0 {
dirs = defaultWatchDirs()
}
ingesters := []ingest.Ingester{
ingest.NewSessionDiffIngester(),
ingest.NewLogIngester(),
ingest.NewSkillsIngester(),
}
var totalNew, totalUpdated int
record := func(result *ingest.IngestResult) {
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
LineStart: result.LineStart,
LineEnd: result.LineEnd,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "ingest",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
return
}
if isNew {
totalNew++
} else {
totalUpdated++
}
if result.SourceID == "opencode-session" {
sessionID, _ := result.Provenance["session_id"].(string)
if sessionID != "" {
kdb.UpsertSession(sessionID, result.Project, result.Title, "active")
}
}
}
for _, ing := range ingesters {
sid := ing.SourceID()
log.Printf("[knox] ingesting %s...", sid)
for _, dir := range dirs {
patterns := []string{
dir + "/*",
dir + "/*/SKILL.md",
}
for _, pattern := range patterns {
entries, _ := filepath.Glob(pattern)
for _, path := range entries {
if !watch.MatchesIngester(path, sid) {
continue
}
result, err := ing.Ingest(path)
if err != nil || result == nil {
continue
}
record(result)
}
}
}
}
// Non-file sources: obsidian vault, browser history, gitea
if vault, err := ingest.DetectObsidianVault(); err == nil {
log.Printf("[knox] ingesting obsidian...")
if results, err := ingest.NewObsidianIngester(vault).IngestAll(); err == nil {
for _, r := range results {
record(r)
}
}
}
log.Printf("[knox] ingesting browser-history...")
if results, err := ingest.NewBrowserHistoryIngester().IngestAll(); err == nil {
for _, r := range results {
record(r)
}
}
log.Printf("[knox] ingesting gitea...")
if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
for _, r := range results {
record(r)
}
}
fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
},
}
cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to scan (default: opencode storage/log + skills)")
return cmd
}
+655
View File
@@ -0,0 +1,655 @@
package cmd
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/index"
"github.com/mark3labs/mcp-go/mcp"
"github.com/mark3labs/mcp-go/server"
)
type detailLevel int
const (
detailBrief detailLevel = iota
detailNormal
detailFull
)
func parseDetail(s string) detailLevel {
switch strings.ToLower(s) {
case "brief":
return detailBrief
case "full":
return detailFull
default:
return detailNormal
}
}
func NewMCPServer(kdb *db.KnoxDB) *server.MCPServer {
s := server.NewMCPServer(
"knox-knowledge",
"0.2.0",
server.WithResourceCapabilities(true, true),
server.WithPromptCapabilities(true),
server.WithLogging(),
)
// ─── knox_search ──────────────────────────────────────────
searchTool := mcp.NewTool("knox_search",
mcp.WithDescription("Search the knowledge index. Supports zoom levels: brief (lean), normal (default), full (verbose). Scope filters: all, skills, sessions, logs."),
mcp.WithString("query", mcp.Required(), mcp.Description("Search query")),
mcp.WithNumber("limit", mcp.Description("Max results (default 10)")),
mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")),
mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")),
)
s.AddTool(searchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
query, _ := req.Params.Arguments["query"].(string)
limit := clampInt(req, "limit", 10, 1, 50)
detail := parseDetail(getString(req, "detail", "normal"))
scope := getString(req, "scope", "all")
results, err := kdb.Search(query, limit)
if err != nil {
return errorResult(err.Error()), nil
}
results = filterByScope(results, scope)
if len(results) == 0 {
return mcp.NewToolResultText(fmt.Sprintf("No results for %q in scope %q", query, scope)), nil
}
return mcp.NewToolResultText(formatEntries(results, detail, kdb)), nil
})
// ─── knox_recent ───────────────────────────────────────────
recentTool := mcp.NewTool("knox_recent",
mcp.WithDescription("Show recent knowledge entries with zoom levels: brief, normal (default), full."),
mcp.WithNumber("limit", mcp.Description("Max results (default 10)")),
mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")),
mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")),
)
s.AddTool(recentTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
limit := clampInt(req, "limit", 10, 1, 50)
detail := parseDetail(getString(req, "detail", "normal"))
scope := getString(req, "scope", "all")
entries, err := kdb.RecentEntries(limit)
if err != nil {
return errorResult(err.Error()), nil
}
entries = filterByScope(entries, scope)
if len(entries) == 0 {
return mcp.NewToolResultText("No entries yet."), nil
}
return mcp.NewToolResultText(formatEntries(entries, detail, kdb)), nil
})
// ─── knox_get ──────────────────────────────────────────────
getTool := mcp.NewTool("knox_get",
mcp.WithDescription("Get full detail for a specific entry by fingerprint (drill-down from search results)."),
mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint (full or first 8 chars)")),
)
s.AddTool(getTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
fp := getString(req, "fingerprint", "")
entry, err := kdb.FindEntry(fp)
if err != nil {
return errorResult(err.Error()), nil
}
if entry == nil {
// Try fingerprint prefix match
entries, err := kdb.FindEntryByPrefix(fp, 5)
if err != nil {
return errorResult(err.Error()), nil
}
if len(entries) == 1 {
entry = &entries[0]
} else if len(entries) > 1 {
return mcp.NewToolResultText(fmt.Sprintf("Multiple matches for %q — use full fingerprint:\n%s", fp, formatEntriesBrief(entries))), nil
} else {
return mcp.NewToolResultText(fmt.Sprintf("No entry found for fingerprint %q", fp)), nil
}
}
// Full detail: entry + observations + provenance chain
var b strings.Builder
b.WriteString(formatEntryFull(entry))
obs, _ := kdb.ObservationsByFingerprint(entry.Fingerprint, 20)
if len(obs) > 0 {
b.WriteString(fmt.Sprintf("\nObservations (%d):\n", len(obs)))
for _, o := range obs {
b.WriteString(fmt.Sprintf(" [%s] trigger=%s confidence=%.1f line=%d ver=%s\n",
o.CollectedAt, o.Trigger, o.Confidence, o.LineEnd, o.IngesterVersion))
}
}
chain, _ := kdb.ProvenanceChain(entry.Fingerprint, 3)
if len(chain) > 0 {
b.WriteString(fmt.Sprintf("\nProvenance links (%d):\n", len(chain)))
for _, l := range chain {
b.WriteString(fmt.Sprintf(" %s --[%s]--> %s (hop %d)\n", shortFP(l.Fingerprint), l.Relation, shortFP(entry.Fingerprint), l.HopDistance))
}
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_stats ────────────────────────────────────────────
statsTool := mcp.NewTool("knox_stats",
mcp.WithDescription("Get knowledge index statistics"),
)
s.AddTool(statsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
stats, err := kdb.Stats()
if err != nil {
return errorResult(err.Error()), nil
}
b, _ := json.MarshalIndent(stats, "", " ")
return mcp.NewToolResultText(string(b)), nil
})
// ─── knox_count ────────────────────────────────────────────
countTool := mcp.NewTool("knox_count",
mcp.WithDescription("Return hit count matching a query, grouped by source. Lightweight — no result bodies, minimal tokens."),
mcp.WithString("query", mcp.Required(), mcp.Description("Search query")),
)
s.AddTool(countTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
query, _ := req.Params.Arguments["query"].(string)
results, err := kdb.Search(query, 500)
if err != nil {
return errorResult(err.Error()), nil
}
total := len(results)
bySource := make(map[string]int)
for _, r := range results {
bySource[r.SourceID]++
}
var b strings.Builder
b.WriteString(fmt.Sprintf("Matches for %q: %d total\n", query, total))
for src, count := range bySource {
b.WriteString(fmt.Sprintf(" %-20s %d\n", src+":", count))
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_reflect ──────────────────────────────────────────
reflectTool := mcp.NewTool("knox_reflect",
mcp.WithDescription("Process pending sessions for reflection and gap analysis"),
)
s.AddTool(reflectTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
reflector := index.NewReflector(kdb)
results, err := reflector.Reflect()
if err != nil {
return errorResult(err.Error()), nil
}
if len(results) == 0 {
return mcp.NewToolResultText("No pending sessions to reflect on."), nil
}
return mcp.NewToolResultText(reflector.FormatResult(results)), nil
})
// ─── knox_thread_list ───────────────────────────────────────
threadListTool := mcp.NewTool("knox_thread_list",
mcp.WithDescription("List knowledge development threads with their motivation, status, and linked entry count."),
mcp.WithString("status", mcp.Description("Filter: active, stalled, resolved (default all)")),
)
s.AddTool(threadListTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
status := getString(req, "status", "")
threads, err := kdb.ListThreads(status)
if err != nil {
return errorResult(err.Error()), nil
}
if len(threads) == 0 {
return mcp.NewToolResultText("No threads."), nil
}
var b strings.Builder
for _, t := range threads {
b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount))
if t.Motivation != "" {
b.WriteString(fmt.Sprintf(" %s\n", t.Motivation))
}
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_thread_create ─────────────────────────────────────
threadCreateTool := mcp.NewTool("knox_thread_create",
mcp.WithDescription("Create a new knowledge development thread to track intent/motivation behind observations."),
mcp.WithString("title", mcp.Required(), mcp.Description("Thread title")),
mcp.WithString("motivation", mcp.Description("Why this thread exists (the 'so what')")),
mcp.WithString("priority", mcp.Description("high, medium, or low (default medium)")),
mcp.WithString("provenance", mcp.Description("JSON provenance metadata: {\"trigger\":\"...\",\"session_id\":\"...\"}")),
)
s.AddTool(threadCreateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
title := getString(req, "title", "")
motivation := getString(req, "motivation", "")
priority := getString(req, "priority", "medium")
provenance := getString(req, "provenance", "{}")
id, err := kdb.CreateThread(title, motivation, priority, "[]", provenance)
if err != nil {
return errorResult(err.Error()), nil
}
return mcp.NewToolResultText(fmt.Sprintf("Created thread #%d: %s", id, title)), nil
})
// ─── knox_thread_update ─────────────────────────────────────
threadUpdateTool := mcp.NewTool("knox_thread_update",
mcp.WithDescription("Rewrite a thread's editorial fields (title/motivation/priority/tags). Used to polish auto-generated threads."),
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID to update")),
mcp.WithString("title", mcp.Description("New title (leave empty to keep)")),
mcp.WithString("motivation", mcp.Description("New motivation (leave empty to keep)")),
mcp.WithString("priority", mcp.Description("New priority: high/medium/low (empty to keep)")),
mcp.WithString("tags", mcp.Description("New space-separated tags (empty to keep)")),
)
s.AddTool(threadUpdateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
id := int64(clampInt(req, "thread_id", 0, 1, 999999))
title := getString(req, "title", "")
motivation := getString(req, "motivation", "")
priority := getString(req, "priority", "")
tags := getString(req, "tags", "")
changed, err := kdb.UpdateThread(id, title, motivation, priority, tags)
if err != nil {
return errorResult(err.Error()), nil
}
if !changed {
return mcp.NewToolResultText(fmt.Sprintf("Thread #%d unchanged (no edits differed).", id)), nil
}
return mcp.NewToolResultText(fmt.Sprintf("Updated thread #%d", id)), nil
})
// ─── knox_thread_draft ──────────────────────────────────────
// Emits auto-threaded candidates with enough context (sources, linked
// observations) for an agent to write tight titles/motivations.
threadDraftTool := mcp.NewTool("knox_thread_draft",
mcp.WithDescription("List auto-threaded candidates needing LLM polish. Returns each heuristic thread's current title/motivation and its linked observations so you can rewrite them via knox_thread_update."),
mcp.WithNumber("limit", mcp.Description("Max candidates to return (default 20)")),
)
s.AddTool(threadDraftTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
limit := clampInt(req, "limit", 20, 1, 200)
threads, err := kdb.ListThreads("")
if err != nil {
return errorResult(err.Error()), nil
}
selected := make([]db.Thread, 0, len(threads))
for _, t := range threads {
if strings.Contains(t.Provenance, "auto_thread") || strings.HasPrefix(t.Motivation, "Auto-detected") {
selected = append(selected, t)
}
if len(selected) >= limit {
break
}
}
if len(selected) == 0 {
return mcp.NewToolResultText("No auto-threaded candidates. Run `knox thread auto` or wait for the watch tick first."), nil
}
var b strings.Builder
for _, t := range selected {
obs, _ := kdb.ThreadObservations(t.ID)
fmt.Fprintf(&b, "THREAD #%d [%s] %d obs\n title: %s\n motivation: %s\n",
t.ID, t.Priority, t.EntryCount, t.Title, t.Motivation)
if t.Tags != "[]" {
fmt.Fprintf(&b, " tags: %s\n", t.Tags)
}
b.WriteString(" linked observations:\n")
for _, o := range obs {
line := strings.TrimSpace(o.Title)
if o.Summary != "" {
line += " — " + strings.TrimSpace(o.Summary)
}
fmt.Fprintf(&b, " • [%s] %s\n", o.SourceID, truncateStr(line, 160))
}
b.WriteString("\n")
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_thread_link ───────────────────────────────────────
threadLinkTool := mcp.NewTool("knox_thread_link",
mcp.WithDescription("Link an observation to a thread by observation ID. Use knox_get first to find observation IDs."),
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")),
mcp.WithNumber("observation_id", mcp.Required(), mcp.Description("Observation ID to link")),
mcp.WithString("relevance", mcp.Description("Why this observation matters to the thread")),
)
s.AddTool(threadLinkTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
threadID := int64(clampInt(req, "thread_id", 0, 1, 999999))
obsID := int64(clampInt(req, "observation_id", 0, 1, 999999))
relevance := getString(req, "relevance", "")
if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil {
return errorResult(err.Error()), nil
}
return mcp.NewToolResultText(fmt.Sprintf("Linked observation #%d to thread #%d", obsID, threadID)), nil
})
// ─── knox_thread_search ─────────────────────────────────────
threadSearchTool := mcp.NewTool("knox_thread_search",
mcp.WithDescription("Search threads by motivation, title, or tags. Find in-flight knowledge work by intent."),
mcp.WithString("query", mcp.Required(), mcp.Description("Search motivation, title, or tags")),
)
s.AddTool(threadSearchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
query := getString(req, "query", "")
threads, err := kdb.SearchThreadsByMotivation(query)
if err != nil {
return errorResult(err.Error()), nil
}
if len(threads) == 0 {
return mcp.NewToolResultText(fmt.Sprintf("No threads matching %q", query)), nil
}
var b strings.Builder
for _, t := range threads {
b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount))
if t.Motivation != "" {
b.WriteString(fmt.Sprintf(" %s\n", t.Motivation))
}
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_thread_link_entry ─────────────────────────────────
threadLinkEntryTool := mcp.NewTool("knox_thread_link_entry",
mcp.WithDescription("Link an entry to a thread via provenance graph. Use after knox_search to connect findings to intent."),
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")),
mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint")),
mcp.WithString("relation", mcp.Description("Relation type: produced (default), references, motivated")),
)
s.AddTool(threadLinkEntryTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
threadID := int64(clampInt(req, "thread_id", 0, 1, 999999))
fp := getString(req, "fingerprint", "")
relation := getString(req, "relation", "produced")
if err := kdb.LinkEntryToThread(threadID, fp, relation); err != nil {
return errorResult(err.Error()), nil
}
return mcp.NewToolResultText(fmt.Sprintf("Linked %s to thread #%d [%s]", shortFP(fp), threadID, relation)), nil
})
// ─── knox_thread_relate ─────────────────────────────────────
threadRelateTool := mcp.NewTool("knox_thread_relate",
mcp.WithDescription("Relate two threads (parent spawns child, threads merge, etc.). Builds the thread-inquiry graph."),
mcp.WithNumber("parent_id", mcp.Required(), mcp.Description("Parent thread ID")),
mcp.WithNumber("child_id", mcp.Required(), mcp.Description("Child thread ID")),
mcp.WithString("relation", mcp.Description("Relation: spawned (default), merged_into, references, supersedes")),
)
s.AddTool(threadRelateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
parentID := int64(clampInt(req, "parent_id", 0, 1, 999999))
childID := int64(clampInt(req, "child_id", 0, 1, 999999))
relation := getString(req, "relation", "spawned")
if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil {
return errorResult(err.Error()), nil
}
return mcp.NewToolResultText(fmt.Sprintf("Related thread #%d -> #%d [%s]", parentID, childID, relation)), nil
})
// ─── knox_thread_golden ─────────────────────────────────────
goldenTool := mcp.NewTool("knox_thread_golden",
mcp.WithDescription("Set, get, or clear the golden thread. The golden thread represents your current focus — new observations auto-link to it."),
mcp.WithNumber("thread_id", mcp.Description("Thread ID to set as golden. Omit to query current. Pass 0 to clear.")),
)
s.AddTool(goldenTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
threadID := int64(clampInt(req, "thread_id", 0, 0, 999999))
if threadID == 0 {
// Query or clear
current, _ := kdb.GoldenThreadID()
if current == 0 {
return mcp.NewToolResultText("No golden thread set."), nil
}
// If thread_id was explicitly 0 and a golden exists, clear it
if _, ok := req.Params.Arguments["thread_id"]; ok {
kdb.SetGoldenThread(0)
t, _ := kdb.GetThread(current)
return mcp.NewToolResultText(fmt.Sprintf("Golden thread cleared (was #%d: %s)", current, t.Title)), nil
}
t, _ := kdb.GetThread(current)
return mcp.NewToolResultText(fmt.Sprintf("Golden thread: #%d %s [%s]\n %s", t.ID, t.Title, t.Status, t.Motivation)), nil
}
if err := kdb.SetGoldenThread(threadID); err != nil {
return errorResult(err.Error()), nil
}
t, _ := kdb.GetThread(threadID)
return mcp.NewToolResultText(fmt.Sprintf("Golden thread set to #%d: %s", threadID, t.Title)), nil
})
// ─── knox_topics ───────────────────────────────────────────
topicsTool := mcp.NewTool("knox_topics",
mcp.WithDescription("List auto-detected topic clusters from TF-IDF analysis. Shows what topics appear across all sources."),
mcp.WithNumber("limit", mcp.Description("Max topics (default 10)")),
)
s.AddTool(topicsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
limit := clampInt(req, "limit", 10, 1, 30)
entries, _ := kdb.RecentEntries(2000)
tfidf := index.BuildTFIDF(entries)
clusters := tfidf.Cluster(2, limit)
if len(clusters) == 0 {
return mcp.NewToolResultText("No topic clusters detected."), nil
}
var b strings.Builder
for i, c := range clusters {
total := 0
for _, v := range c.BySource {
total += v
}
b.WriteString(fmt.Sprintf("%d. %s (%d entries)\n", i+1, c.Name, total))
var sources []string
for src, count := range c.BySource {
sources = append(sources, fmt.Sprintf("%s:%d", src, count))
}
b.WriteString(fmt.Sprintf(" Sources: %s\n", strings.Join(sources, ", ")))
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_gaps ─────────────────────────────────────────────
gapsTool := mcp.NewTool("knox_gaps",
mcp.WithDescription("Find topics with browser activity but no skill coverage. Identifies candidates for new skill creation."),
)
s.AddTool(gapsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
entries, _ := kdb.RecentEntries(2000)
skillKeywords := make(map[string]bool)
for _, e := range entries {
if e.SourceID == "skills-catalog" {
for _, w := range index.Tokenize(e.Title + " " + e.Summary) {
skillKeywords[w] = true
}
}
}
tfidf := index.BuildTFIDF(entries)
clusters := tfidf.Cluster(2, 15)
gaps := tfidf.Gaps(clusters, skillKeywords)
if len(gaps) == 0 {
return mcp.NewToolResultText("No gaps found."), nil
}
var b strings.Builder
b.WriteString(fmt.Sprintf("Found %d topics with browser activity but NO skill coverage:\n\n", len(gaps)))
for i, g := range gaps {
browserCount := g.BySource["browser-history"]
b.WriteString(fmt.Sprintf("%d. %s — %d browser visits\n", i+1, g.Name, browserCount))
}
return mcp.NewToolResultText(b.String()), nil
})
// ─── knox_git_status ───────────────────────────────────────
gitStatusTool := mcp.NewTool("knox_git_status",
mcp.WithDescription("List local git repository status captured by Knox (branch, clean/dirty, ahead/behind, HEAD, last commit). Pass project to filter."),
mcp.WithString("project", mcp.Description("Optional project/repo name substring filter")),
)
s.AddTool(gitStatusTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
entries, err := kdb.EntriesBySource("git")
if err != nil {
return errorResult(err.Error()), nil
}
filter, _ := req.Params.Arguments["project"].(string)
if len(entries) == 0 {
return mcp.NewToolResultText("No git status entries yet. Run `knox git` or wait for the daemon's git timer."), nil
}
var b strings.Builder
shown := 0
for _, e := range entries {
if filter != "" && !strings.Contains(strings.ToLower(e.Project), strings.ToLower(filter)) {
continue
}
b.WriteString(fmt.Sprintf("%-24s %s\n", e.Project, e.Summary))
shown++
}
b.WriteString(fmt.Sprintf("\n%d / %d repos", shown, len(entries)))
return mcp.NewToolResultText(b.String()), nil
})
return s
}
// ─── Format helpers ──────────────────────────────────────────
func formatEntries(entries []db.Entry, detail detailLevel, kdb *db.KnoxDB) string {
switch detail {
case detailBrief:
return formatEntriesBrief(entries)
case detailFull:
var b strings.Builder
for i, e := range entries {
if i > 0 {
b.WriteString("---\n")
}
b.WriteString(formatEntryFull(&e))
}
return b.String()
default:
var b strings.Builder
for _, e := range entries {
b.WriteString(formatEntryNormal(&e))
b.WriteString("\n")
}
return b.String()
}
}
func formatEntriesBrief(entries []db.Entry) string {
var b strings.Builder
for _, e := range entries {
b.WriteString(fmt.Sprintf("%-8s | %-40s | %-14s | %s\n",
shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project))
}
return b.String()
}
func formatEntryNormal(e *db.Entry) string {
return fmt.Sprintf("%-8s %-40s [%-14s %-4s] seen:%d conf:%.1f\n %s",
shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project,
e.RefCount, e.LastConfidence, truncateStr(e.Summary, 90))
}
func formatEntryFull(e *db.Entry) string {
return fmt.Sprintf(`fingerprint: %s
title: %s
source: %s
path: %s
project: %s
type: %s
first_seen: %s
last_seen: %s
ref_count: %d
confidence: %.1f
summary: %s`,
e.Fingerprint, e.Title, e.SourceID, e.SourcePath, e.Project, e.ContentType,
e.FirstSeen, e.LastSeen, e.RefCount, e.LastConfidence, e.Summary)
}
// ─── Scope filtering ─────────────────────────────────────────
func filterByScope(entries []db.Entry, scope string) []db.Entry {
switch strings.ToLower(scope) {
case "skills":
return filterSource(entries, "skills-catalog")
case "sessions":
return filterSource(entries, "opencode-session")
case "logs":
return filterSource(entries, "opencode-log")
default:
return entries
}
}
func filterSource(entries []db.Entry, sourceID string) []db.Entry {
var filtered []db.Entry
for _, e := range entries {
if e.SourceID == sourceID {
filtered = append(filtered, e)
}
}
return filtered
}
// ─── Argument helpers ────────────────────────────────────────
func getString(req mcp.CallToolRequest, key, def string) string {
if v, ok := req.Params.Arguments[key].(string); ok {
return v
}
return def
}
func clampInt(req mcp.CallToolRequest, key string, def, min, max int) int {
if v, ok := req.Params.Arguments[key].(float64); ok {
n := int(v)
if n < min {
return min
}
if n > max {
return max
}
return n
}
return def
}
func shortFP(fp string) string {
if len(fp) > 8 {
return fp[:8]
}
return fp
}
func truncateStr(s string, n int) string {
runes := []rune(s)
if len(runes) <= n {
return s
}
return string(runes[:n]) + "..."
}
func errorResult(msg string) *mcp.CallToolResult {
return &mcp.CallToolResult{
IsError: true,
Content: []mcp.Content{mcp.NewTextContent(msg)},
}
}
+71
View File
@@ -0,0 +1,71 @@
package cmd
import (
"fmt"
"log"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
)
func NewObsidianCmd(kdb *db.KnoxDB) *cobra.Command {
var vault string
cmd := &cobra.Command{
Use: "obsidian",
Short: "Ingest Obsidian notes into the knowledge index",
Long: `Scans an Obsidian vault for markdown notes, extracts frontmatter
(title, tags, dates), and fingerprints them into the index.
Auto-detects vault path from ~/.config/obsidian/obsidian.json
or override with --vault.`,
RunE: func(c *cobra.Command, args []string) error {
ing := ingest.NewObsidianIngester(vault)
log.Printf("[knox] ingesting Obsidian notes...")
results, err := ing.IngestAll()
if err != nil {
return fmt.Errorf("obsidian ingest: %w", err)
}
if len(results) == 0 {
fmt.Println("No Obsidian notes found.")
return nil
}
var newCount, dupCount int
for _, result := range results {
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
LineStart: result.LineStart,
LineEnd: result.LineEnd,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "obsidian_ingest",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
continue
}
if isNew {
newCount++
} else {
dupCount++
}
}
fmt.Printf("Obsidian notes: %d new, %d updated\n", newCount, dupCount)
return nil
},
}
cmd.Flags().StringVarP(&vault, "vault", "v", "", "Obsidian vault path (auto-detect if empty)")
return cmd
}
+161
View File
@@ -0,0 +1,161 @@
package cmd
import (
"encoding/json"
"fmt"
"os"
"strings"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/index"
)
func paginate(entries []db.Entry, page, limit int) []db.Entry {
start := (page - 1) * limit
if start >= len(entries) {
return nil
}
end := start + limit
if end > len(entries) {
end = len(entries)
}
return entries[start:end]
}
func NewStatusCmd(kdb *db.KnoxDB) *cobra.Command {
var jsonOut bool
cmd := &cobra.Command{
Use: "status",
Short: "Show knowledge index statistics",
RunE: func(c *cobra.Command, args []string) error {
stats, err := kdb.Stats()
if err != nil {
return err
}
if jsonOut {
b, _ := json.MarshalIndent(stats, "", " ")
fmt.Println(string(b))
return nil
}
fmt.Println("Knox Knowledge Index")
fmt.Println("====================")
for k, v := range stats {
switch val := v.(type) {
case int:
fmt.Printf(" %-24s %d\n", k+":", val)
case string:
fmt.Printf(" %-24s %s\n", k+":", val)
default:
fmt.Printf(" %-24s %v\n", k+":", val)
}
}
return nil
},
}
cmd.Flags().BoolVar(&jsonOut, "json", false, "Output as JSON")
return cmd
}
func NewQueryCmd(kdb *db.KnoxDB) *cobra.Command {
var page, limit int
cmd := &cobra.Command{
Use: "query [search text]",
Short: "Search the knowledge index",
Long: "Search the knowledge index. Reads from stdin pipe if no positional arg given.",
RunE: func(c *cobra.Command, args []string) error {
query := ""
if len(args) > 0 {
query = args[0]
} else {
stdin, _ := os.ReadFile(os.Stdin.Name())
query = strings.TrimSpace(string(stdin))
}
if query == "" {
return fmt.Errorf("search query required (positional arg or stdin pipe)")
}
results, err := kdb.Search(query, limit)
if err != nil {
return err
}
results = paginate(results, page, limit)
if len(results) == 0 {
fmt.Println("No results found.")
return nil
}
fmt.Printf("Found %d results for %q (page %d, %d per page):\n\n", len(results), query, page, limit)
for _, r := range results {
fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(r.Fingerprint), truncateStr(r.Title, 30), r.Project, r.SourceID)
if r.Summary != "" {
fmt.Printf(" %-8s %s\n", "", truncateStr(r.Summary, 80))
}
fmt.Println()
}
return nil
},
}
cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)")
cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page")
return cmd
}
func NewRecentCmd(kdb *db.KnoxDB) *cobra.Command {
var page, limit int
cmd := &cobra.Command{
Use: "recent",
Short: "Show recent knowledge entries",
RunE: func(c *cobra.Command, args []string) error {
all, err := kdb.RecentEntries(limit * 10)
if err != nil {
return err
}
entries := paginate(all, page, limit)
if len(entries) == 0 {
fmt.Println("No entries yet. Run `knox watch` to start ingesting.")
return nil
}
fmt.Printf("Recent %d entries (page %d, %d per page):\n\n", len(entries), page, limit)
for _, e := range entries {
fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(e.Fingerprint), truncateStr(e.Title, 30), e.Project, e.SourceID)
}
return nil
},
}
cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)")
cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page")
return cmd
}
func NewReflectCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "reflect",
Short: "Process pending sessions for reflection",
Long: "Scans for sessions not yet indexed, runs reflection, and reports findings.",
RunE: func(c *cobra.Command, args []string) error {
reflector := index.NewReflector(kdb)
results, err := reflector.Reflect()
if err != nil {
return err
}
if len(results) == 0 {
fmt.Println("All caught up — no pending sessions to reflect on.")
return nil
}
fmt.Printf("Reflected %d sessions:\n\n", len(results))
fmt.Print(reflector.FormatResult(results))
gaps, _ := reflector.Gaps()
if len(gaps) > 0 {
fmt.Printf("\nProjects with indexed knowledge:\n")
for _, g := range gaps {
fmt.Printf(" %-20s %d entries\n", g.Project, g.TotalEntries)
}
}
return nil
},
}
}
func getHomeDir() (string, error) {
return os.UserHomeDir()
}
+471
View File
@@ -0,0 +1,471 @@
package cmd
import (
"fmt"
"strconv"
"strings"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/watch"
"github.com/spf13/cobra"
)
func NewThreadCmd(kdb *db.KnoxDB) *cobra.Command {
cmd := &cobra.Command{
Use: "thread",
Short: "Manage knowledge development threads (motivation/intent tracking)",
Long: `Threads capture WHY observations were collected — the intent,
motivation, and in-flight knowledge development work.
Subcommands: create, list, link, link-entry, relate, golden, prune, rescan, close, show, note`,
}
cmd.AddCommand(newThreadCreateCmd(kdb))
cmd.AddCommand(newThreadAutoCmd(kdb))
cmd.AddCommand(newThreadDraftCmd(kdb))
cmd.AddCommand(newThreadUpdateCmd(kdb))
cmd.AddCommand(newThreadListCmd(kdb))
cmd.AddCommand(newThreadLinkCmd(kdb))
cmd.AddCommand(newThreadCloseCmd(kdb))
cmd.AddCommand(newThreadShowCmd(kdb))
cmd.AddCommand(newThreadNoteCmd(kdb))
cmd.AddCommand(newThreadLinkEntryCmd(kdb))
cmd.AddCommand(newThreadRelateCmd(kdb))
cmd.AddCommand(newThreadGoldenCmd(kdb))
cmd.AddCommand(newThreadPruneCmd(kdb))
cmd.AddCommand(newThreadRescanCmd(kdb))
return cmd
}
func newThreadCreateCmd(kdb *db.KnoxDB) *cobra.Command {
var motivation, priority, tags, provenance string
cmd := &cobra.Command{
Use: "create <title>",
Short: "Create a new knowledge development thread",
Args: cobra.MinimumNArgs(1),
RunE: func(c *cobra.Command, args []string) error {
title := strings.Join(args, " ")
id, err := kdb.CreateThread(title, motivation, priority, tags, provenance)
if err != nil {
return err
}
fmt.Printf("Created thread #%d: %s\n", id, title)
return nil
},
}
cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "Why this thread exists (the 'so what')")
cmd.Flags().StringVarP(&priority, "priority", "p", "medium", "Priority: high, medium, low")
cmd.Flags().StringVarP(&tags, "tags", "t", "", "Comma-separated tags")
cmd.Flags().StringVarP(&provenance, "provenance", "", "{}", "JSON provenance metadata")
return cmd
}
func newThreadAutoCmd(kdb *db.KnoxDB) *cobra.Command {
var dryRun bool
cmd := &cobra.Command{
Use: "auto",
Short: "Auto-create threads from high-intent topic clusters (heuristic, no LLM)",
Long: `Clusters recent observations with TF-IDF and promotes clusters that cross
the "intent bar" into threads, auto-linking their observations.
The intent bar (tunable via KNOX_THREAD_* env vars) requires: enough distinct
sources, at least one non-passive work source (git/gitea/session/obsidian), and
observations observed recently. Use --dry-run to preview without writing.`,
RunE: func(c *cobra.Command, args []string) error {
threader := watch.NewAutoThreader(kdb)
threader.DryRun = dryRun
created, linked, err := threader.AutoThread()
if err != nil {
return err
}
fmt.Printf("\nAuto-thread: %d created, %d observations linked\n", created, linked)
return nil
},
}
cmd.Flags().BoolVar(&dryRun, "dry-run", dryRun, "Preview candidate threads without writing")
return cmd
}
func newThreadDraftCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "draft",
Short: "List auto-threaded candidates for LLM title/motivation polish",
Long: `Shows threads created by the heuristic auto-threader ('auto_thread'
provenance) alongside their linked observations, so an LLM or editor can write
a better title and motivation. Apply changes with ` + "`knox thread update <id>`" + `.`,
RunE: func(c *cobra.Command, args []string) error {
threads, err := kdb.ListThreads("")
if err != nil {
return err
}
var shown int
for _, t := range threads {
if !isAutoThreaded(t) {
continue
}
obs, _ := kdb.ThreadObservations(t.ID)
fmt.Printf("Thread #%d [%s] %d obs\n", t.ID, t.Priority, t.EntryCount)
fmt.Printf(" title: %s\n", t.Title)
fmt.Printf(" motivation: %s\n", t.Motivation)
if t.Tags != "[]" {
fmt.Printf(" tags: %s\n", t.Tags)
}
fmt.Println(" linked observations:")
for _, o := range obs {
title := truncateStr(o.Title, 70)
if o.Summary != "" {
title += " — " + truncateStr(o.Summary, 60)
}
fmt.Printf(" • [%s] %s\n", o.SourceID, title)
}
fmt.Println()
shown++
}
if shown == 0 {
fmt.Println("No auto-threaded candidates found. Run `knox thread auto` (or wait for the watch tick) first.")
}
return nil
},
}
}
func isAutoThreaded(t db.Thread) bool {
if strings.Contains(t.Provenance, "auto_thread") {
return true
}
return strings.HasPrefix(t.Motivation, "Auto-detected")
}
func newThreadUpdateCmd(kdb *db.KnoxDB) *cobra.Command {
var title, motivation, priority, tags string
cmd := &cobra.Command{
Use: "update <thread-id>",
Short: "Rewrite a thread's title/motivation/priority/tags",
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
changed, err := kdb.UpdateThread(id, title, motivation, priority, tags)
if err != nil {
return err
}
if !changed {
fmt.Println("No changes applied.")
return nil
}
fmt.Printf("Updated thread #%d\n", id)
return nil
},
}
cmd.Flags().StringVarP(&title, "title", "t", "", "New title")
cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "New motivation")
cmd.Flags().StringVarP(&priority, "priority", "p", "", "New priority (high/medium/low)")
cmd.Flags().StringVar(&tags, "tags", "", "New space-separated tags")
return cmd
}
func newThreadListCmd(kdb *db.KnoxDB) *cobra.Command {
var status string
cmd := &cobra.Command{
Use: "list",
Short: "List knowledge development threads",
RunE: func(c *cobra.Command, args []string) error {
threads, err := kdb.ListThreads(status)
if err != nil {
return err
}
if len(threads) == 0 {
fmt.Println("No threads. Create one with `knox thread create`.")
return nil
}
fmt.Printf("%-4s %-40s %-10s %-8s %s\n", "ID", "Title", "Status", "Priority", "Entries")
fmt.Println(strings.Repeat("-", 75))
for _, t := range threads {
fmt.Printf("%-4d %-40s %-10s %-8s %d\n", t.ID, truncateStr(t.Title, 38), t.Status, t.Priority, t.EntryCount)
if t.Motivation != "" {
fmt.Printf(" %s\n", truncateStr(t.Motivation, 70))
}
}
return nil
},
}
cmd.Flags().StringVarP(&status, "status", "s", "", "Filter by status (active, stalled, resolved)")
return cmd
}
func newThreadLinkCmd(kdb *db.KnoxDB) *cobra.Command {
var relevance string
cmd := &cobra.Command{
Use: "link <thread-id> <observation-id>",
Short: "Link an observation to a thread",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
threadID, _ := strconv.ParseInt(args[0], 10, 64)
obsID, _ := strconv.ParseInt(args[1], 10, 64)
if threadID == 0 || obsID == 0 {
return fmt.Errorf("thread-id and observation-id must be numeric")
}
if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil {
return err
}
fmt.Printf("Linked observation #%d to thread #%d\n", obsID, threadID)
return nil
},
}
cmd.Flags().StringVarP(&relevance, "relevance", "r", "", "Why this observation matters to the thread")
return cmd
}
func newThreadCloseCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "close <thread-id>",
Short: "Mark a thread as resolved",
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
thread, _ := kdb.GetThread(id)
if thread == nil {
return fmt.Errorf("thread #%d not found", id)
}
if err := kdb.CloseThread(id); err != nil {
return err
}
fmt.Printf("Closed thread #%d: %s\n", id, thread.Title)
return nil
},
}
}
func newThreadShowCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "show <thread-id>",
Short: "Show thread details with provenance, linked observations, and notes",
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
t, err := kdb.GetThread(id)
if err != nil {
return err
}
if t == nil {
return fmt.Errorf("thread #%d not found", id)
}
goldenID, _ := kdb.GoldenThreadID()
fmt.Printf("Thread #%d: %s", t.ID, t.Title)
if goldenID == id {
fmt.Print(" [GOLDEN]")
}
fmt.Println()
fmt.Printf(" Status: %s\n", t.Status)
fmt.Printf(" Priority: %s\n", t.Priority)
fmt.Printf(" Created: %s\n", t.CreatedAt)
fmt.Printf(" Updated: %s\n", t.UpdatedAt)
if t.ResolvedAt != "" {
fmt.Printf(" Resolved: %s\n", t.ResolvedAt)
}
if t.Motivation != "" {
fmt.Printf(" Motivation: %s\n", t.Motivation)
}
if t.Tags != "[]" {
fmt.Printf(" Tags: %s\n", t.Tags)
}
if t.Provenance != "{}" {
fmt.Printf(" Provenance: %s\n", t.Provenance)
}
obs, _ := kdb.ThreadObservations(id)
if len(obs) > 0 {
fmt.Printf("\nLinked observations (%d):\n", len(obs))
for _, o := range obs {
fmt.Printf(" #%-4d %-30s [%s] trigger=%s\n", o.ID, truncateStr(o.Title, 28), o.SourceID, o.Trigger)
}
}
provenance, _ := kdb.ThreadProvenance(id)
if len(provenance) > 0 {
fmt.Printf("\nProvenance graph (%d links):\n", len(provenance))
for _, p := range provenance {
fmt.Printf(" %s --[%s]--> thread:%d\n", shortFP(p.Fingerprint), p.Relation, id)
}
}
notes, _ := kdb.ThreadNotes(id)
if len(notes) > 0 {
fmt.Printf("\nNotes (%d):\n", len(notes))
for _, n := range notes {
fmt.Printf(" [%s] %s\n", n.CreatedAt, n.Note)
}
}
return nil
},
}
}
func newThreadLinkEntryCmd(kdb *db.KnoxDB) *cobra.Command {
var relation string
cmd := &cobra.Command{
Use: "link-entry <thread-id> <entry-fingerprint>",
Short: "Link an entry to a thread via provenance graph",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
if err := kdb.LinkEntryToThread(id, args[1], relation); err != nil {
return err
}
fmt.Printf("Linked entry %s to thread #%d [%s]\n", shortFP(args[1]), id, relation)
return nil
},
}
cmd.Flags().StringVarP(&relation, "relation", "r", "produced", "Relation type (produced, references, motivated)")
return cmd
}
func newThreadRelateCmd(kdb *db.KnoxDB) *cobra.Command {
var relation string
cmd := &cobra.Command{
Use: "relate <parent-id> <child-id>",
Short: "Relate two threads (parent spawned child, etc.)",
Args: cobra.ExactArgs(2),
RunE: func(c *cobra.Command, args []string) error {
parentID, _ := strconv.ParseInt(args[0], 10, 64)
childID, _ := strconv.ParseInt(args[1], 10, 64)
if parentID == 0 || childID == 0 {
return fmt.Errorf("both thread IDs must be numeric")
}
if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil {
return err
}
fmt.Printf("Related thread #%d -> #%d [%s]\n", parentID, childID, relation)
return nil
},
}
cmd.Flags().StringVarP(&relation, "relation", "r", "spawned", "Relation type (spawned, merged_into, references, supersedes)")
return cmd
}
func newThreadGoldenCmd(kdb *db.KnoxDB) *cobra.Command {
var unset bool
cmd := &cobra.Command{
Use: "golden [thread-id]",
Short: "Set/show/clear the golden thread (current focus). Auto-links new observations.",
RunE: func(c *cobra.Command, args []string) error {
if unset {
kdb.SetGoldenThread(0)
fmt.Println("Golden thread cleared.")
return nil
}
if len(args) > 0 {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
if err := kdb.SetGoldenThread(id); err != nil {
return err
}
t, _ := kdb.GetThread(id)
if t != nil {
fmt.Printf("Golden thread set to #%d: %s\n", id, t.Title)
}
return nil
}
goldenID, _ := kdb.GoldenThreadID()
if goldenID == 0 {
fmt.Println("No golden thread set.")
return nil
}
t, _ := kdb.GetThread(goldenID)
if t != nil {
fmt.Printf("Golden thread: #%d %s [%s/%s]\n %s\n", t.ID, t.Title, t.Status, t.Priority, t.Motivation)
}
return nil
},
}
cmd.Flags().BoolVar(&unset, "unset", false, "Clear the golden thread")
return cmd
}
func newThreadNoteCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "note <thread-id> <text>",
Short: "Add a note to a thread",
Args: cobra.MinimumNArgs(2),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
note := strings.Join(args[1:], " ")
nid, err := kdb.AddThreadNote(id, note)
if err != nil {
return err
}
fmt.Printf("Added note #%d to thread #%d\n", nid, id)
return nil
},
}
}
func newThreadPruneCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "prune <thread-id>",
Short: "Remove irrelevant auto-linked observations from a thread",
Long: `Scans auto-linked observations and removes ones that don't share keywords with the thread's title, motivation, or tags.`,
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
t, err := kdb.GetThread(id)
if err != nil || t == nil {
return fmt.Errorf("thread #%d not found", id)
}
keywords, bigrams, err := kdb.ThreadRelevanceProfile(id)
if err != nil {
return err
}
if len(keywords) == 0 {
return fmt.Errorf("no significant keywords found in thread")
}
count, err := kdb.PruneThread(id)
if err != nil {
return err
}
fmt.Printf("Pruned %d irrelevant links from thread #%d (%s)\n", count, id, t.Title)
fmt.Printf(" Kept observations matching: %v (bigrams: %v)\n", keywords, bigrams)
return nil
},
}
}
func newThreadRescanCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "rescan <thread-id>",
Short: "Link unlinked observations that pass the thread's relevance profile",
Long: `Scans all observations and links ones matching the thread's DF-filtered keyword profile. Inverse of prune.`,
Args: cobra.ExactArgs(1),
RunE: func(c *cobra.Command, args []string) error {
id, _ := strconv.ParseInt(args[0], 10, 64)
if id == 0 {
return fmt.Errorf("thread-id must be numeric")
}
t, err := kdb.GetThread(id)
if err != nil || t == nil {
return fmt.Errorf("thread #%d not found", id)
}
count, err := kdb.RescanThread(id)
if err != nil {
return err
}
fmt.Printf("Linked %d observations to thread #%d (%s)\n", count, id, t.Title)
return nil
},
}
}
+135
View File
@@ -0,0 +1,135 @@
package cmd
import (
"fmt"
"regexp"
"strings"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/index"
)
func NewTopicsCmd(kdb *db.KnoxDB) *cobra.Command {
cmd := &cobra.Command{
Use: "topics",
Short: "Auto-detect topic clusters from TF-IDF analysis of the knowledge index",
Long: `Scans all entries, computes TF-IDF term scores, and clusters related
observations into topics. Shows gap analysis for browser-researched
topics not covered by skills.
Subcommands: list, gaps`,
}
cmd.AddCommand(newTopicListCmd(kdb))
cmd.AddCommand(newTopicGapsCmd(kdb))
return cmd
}
func newTopicListCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "list",
Short: "List auto-detected topic clusters",
RunE: func(c *cobra.Command, args []string) error {
entries, err := kdb.RecentEntries(2000)
if err != nil {
return err
}
tfidf := index.BuildTFIDF(entries)
clusters := tfidf.Cluster(2, 15)
if len(clusters) == 0 {
fmt.Println("No topic clusters detected.")
return nil
}
fmt.Printf("%-3s %-50s %-8s %s\n", "ID", "Topic", "Entries", "Sources")
fmt.Println(strings.Repeat("-", 75))
for i, c := range clusters {
total := 0
for _, v := range c.BySource {
total += v
}
sources := ""
for src, count := range c.BySource {
if sources != "" {
sources += ", "
}
sources += fmt.Sprintf("%s:%d", shortSource(src), count)
}
fmt.Printf("%-3d %-50s %-8d %s\n", i+1, truncateStr(c.Name, 48), total, sources)
}
return nil
},
}
}
func newTopicGapsCmd(kdb *db.KnoxDB) *cobra.Command {
return &cobra.Command{
Use: "gaps",
Short: "Show topics with browser activity but no skill coverage",
Long: `Cross-references browser history against the skills catalog.
Flags topics that have significant browser research but zero skill entries.
These are candidates for new skill creation.`,
RunE: func(c *cobra.Command, args []string) error {
entries, err := kdb.RecentEntries(2000)
if err != nil {
return err
}
// Build skill keyword set
skillKeywords := make(map[string]bool)
for _, e := range entries {
if e.SourceID == "skills-catalog" {
for _, w := range tokenize(e.Title + " " + e.Summary) {
skillKeywords[w] = true
}
}
}
tfidf := index.BuildTFIDF(entries)
clusters := tfidf.Cluster(2, 15)
gaps := tfidf.Gaps(clusters, skillKeywords)
if len(gaps) == 0 {
fmt.Println("No gaps found — all browser-researched topics have skill coverage.")
return nil
}
fmt.Printf("\nFound %d topics with browser activity but NO skill coverage:\n\n", len(gaps))
for i, g := range gaps {
browserCount := g.BySource["browser-history"]
fmt.Printf("%d. %s\n", i+1, g.Name)
fmt.Printf(" %d browser visits · keywords: %v\n", browserCount, g.Keywords)
fmt.Println()
}
fmt.Println("Consider creating skills for these topics with `knox asses` or via opencode.")
return nil
},
}
}
func shortSource(s string) string {
switch s {
case "opencode-session":
return "session"
case "skills-catalog":
return "skill"
case "browser-history":
return "browser"
case "obsidian":
return "notes"
case "opencode-log":
return "logs"
}
return s
}
func tokenize(text string) []string {
re := regexp.MustCompile(`[a-z]{4,}`)
return re.FindAllString(strings.ToLower(text), -1)
}
+44
View File
@@ -0,0 +1,44 @@
package cmd
import (
"io"
"log"
"github.com/spf13/cobra"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/watch"
)
func NewWatchCmd(kdb *db.KnoxDB) *cobra.Command {
var dirs []string
var quiet bool
cmd := &cobra.Command{
Use: "watch",
Short: "Watch directories for new session data and ingest into index",
Run: func(cmd *cobra.Command, args []string) {
if len(dirs) == 0 {
dirs = defaultWatchDirs()
}
if quiet {
log.SetOutput(io.Discard)
}
log.Printf("[knox] starting watcher, scanning %d directories", len(dirs))
w := watch.New(kdb, dirs)
if err := w.Start(); err != nil {
log.Fatalf("watcher error: %v", err)
}
},
}
cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to watch (default: opencode storage/log + skills)")
cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Suppress per-entry log output")
return cmd
}
func defaultWatchDirs() []string {
home, _ := getHomeDir()
return []string{
home + "/.local/share/opencode/storage/session_diff",
home + "/.local/share/opencode/log",
home + "/.skills",
}
}
+1325
View File
File diff suppressed because it is too large Load Diff
+135
View File
@@ -0,0 +1,135 @@
package db
const Schema = `
-- IMMUTABLE LOG: every observation is append-only, never mutated
CREATE TABLE IF NOT EXISTS observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
fingerprint TEXT NOT NULL,
source_id TEXT NOT NULL,
source_path TEXT,
project TEXT,
content_type TEXT,
title TEXT,
summary TEXT,
collected_at TEXT NOT NULL DEFAULT (datetime('now')),
created_at TEXT,
line_start INTEGER,
line_end INTEGER,
confidence REAL DEFAULT 0.5,
ingester_version TEXT DEFAULT '1',
trigger TEXT,
provenance TEXT DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS idx_obs_fingerprint ON observations(fingerprint);
CREATE INDEX IF NOT EXISTS idx_obs_collected ON observations(collected_at);
CREATE INDEX IF NOT EXISTS idx_obs_created ON observations(created_at);
CREATE INDEX IF NOT EXISTS idx_obs_source ON observations(source_id, source_path);
CREATE INDEX IF NOT EXISTS idx_obs_project ON observations(project);
CREATE INDEX IF NOT EXISTS idx_obs_trigger ON observations(trigger);
-- MATERIALIZED CACHE: current state, reconstructable from observations
CREATE TABLE IF NOT EXISTS entries (
fingerprint TEXT PRIMARY KEY,
source_id TEXT NOT NULL,
source_path TEXT,
project TEXT,
content_type TEXT,
title TEXT,
summary TEXT,
first_seen TEXT,
last_seen TEXT,
ref_count INTEGER DEFAULT 1,
last_confidence REAL DEFAULT 0.5
);
CREATE INDEX IF NOT EXISTS idx_entries_project ON entries(project);
CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source_id);
CREATE INDEX IF NOT EXISTS idx_entries_seen ON entries(last_seen);
-- PROVENANCE LINKS: multi-hop lineage between entries
CREATE TABLE IF NOT EXISTS provenance_links (
id INTEGER PRIMARY KEY AUTOINCREMENT,
descendant_fp TEXT NOT NULL,
ancestor_fp TEXT NOT NULL,
hop_distance INTEGER NOT NULL,
relation TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_pl_descendant ON provenance_links(descendant_fp);
CREATE INDEX IF NOT EXISTS idx_pl_ancestor ON provenance_links(ancestor_fp);
-- REFLECTION STATE: per-session tracking
CREATE TABLE IF NOT EXISTS sessions (
session_id TEXT PRIMARY KEY,
project TEXT,
title TEXT,
status TEXT DEFAULT 'active',
agent_count INTEGER DEFAULT 0,
started_at TEXT,
ended_at TEXT,
last_seen TEXT DEFAULT (datetime('now')),
indexed INTEGER DEFAULT 0
);
-- SOURCE REGISTRY
CREATE TABLE IF NOT EXISTS sources (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
enabled INTEGER DEFAULT 1,
last_event TEXT,
created_at TEXT DEFAULT (datetime('now'))
);
INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-session', 'Opencode Session Diffs');
INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-log', 'Opencode Log Files');
INSERT OR IGNORE INTO sources (id, name) VALUES ('git', 'Git History');
INSERT OR IGNORE INTO sources (id, name) VALUES ('filesystem', 'File System Watcher');
-- KNOWLEDGE THREADS: why observations exist, what questions they answer
CREATE TABLE IF NOT EXISTS threads (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
motivation TEXT,
status TEXT DEFAULT 'active',
priority TEXT DEFAULT 'medium',
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
resolved_at TEXT,
tags TEXT DEFAULT '[]',
provenance TEXT DEFAULT '{}'
);
-- SETTINGS: key-value store for runtime configuration
CREATE TABLE IF NOT EXISTS settings (
key TEXT PRIMARY KEY,
value TEXT
);
CREATE INDEX IF NOT EXISTS idx_threads_status ON threads(status);
CREATE TABLE IF NOT EXISTS thread_observations (
thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE,
observation_id INTEGER NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
relevance TEXT,
added_at TEXT DEFAULT (datetime('now')),
PRIMARY KEY (thread_id, observation_id)
);
CREATE TABLE IF NOT EXISTS thread_notes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE,
note TEXT NOT NULL,
created_at TEXT DEFAULT (datetime('now'))
);
-- TOPICS: TF-IDF clustered groups of related observations
CREATE TABLE IF NOT EXISTS topics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
keywords TEXT,
score REAL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now'))
);
`
+100
View File
@@ -0,0 +1,100 @@
package index
import (
"fmt"
"strings"
"github.com/david/knox/internal/db"
)
type Reflector struct {
db *db.KnoxDB
}
func NewReflector(kdb *db.KnoxDB) *Reflector {
return &Reflector{db: kdb}
}
type ReflectionResult struct {
SessionID string
Project string
EntryCount int
Status string
}
func (r *Reflector) Reflect() ([]ReflectionResult, error) {
sessions, err := r.db.PendingSessions()
if err != nil {
return nil, fmt.Errorf("pending sessions: %w", err)
}
var results []ReflectionResult
for _, s := range sessions {
entries, err := r.db.EntriesByProject(s.Project, 100)
if err != nil {
return nil, err
}
result := ReflectionResult{
SessionID: s.SessionID,
Project: s.Project,
EntryCount: len(entries),
Status: "reflected",
}
if err := r.db.MarkSessionIndexed(s.SessionID); err != nil {
return nil, err
}
results = append(results, result)
}
return results, nil
}
type GapReport struct {
Project string
TotalEntries int
}
func (r *Reflector) Gaps() ([]GapReport, error) {
entries, err := r.db.RecentEntries(500)
if err != nil {
return nil, err
}
projects := make(map[string]int)
for _, e := range entries {
if e.Project != "" {
projects[e.Project]++
}
}
var reports []GapReport
for proj, count := range projects {
reports = append(reports, GapReport{
Project: proj,
TotalEntries: count,
})
}
return reports, nil
}
func (r *Reflector) FormatResult(results []ReflectionResult) string {
var b strings.Builder
for _, res := range results {
b.WriteString(fmt.Sprintf(" %-12s %-20s %d entries [%s]\n",
truncateStr(res.SessionID, 12),
res.Project,
res.EntryCount,
res.Status,
))
}
return b.String()
}
func truncateStr(s string, n int) string {
runes := []rune(s)
if len(runes) <= n {
return s
}
return string(runes[:n]) + "..."
}
+258
View File
@@ -0,0 +1,258 @@
package index
import (
"math"
"regexp"
"sort"
"strings"
"github.com/david/knox/internal/db"
)
type TopicCluster struct {
Name string
Keywords []string
Score float64
Entries []db.Entry
BySource map[string]int
}
var tokenRE = regexp.MustCompile(`[a-z]{4,}`)
var hexTokenRE = regexp.MustCompile(`^[a-f0-9]{8,}$`)
func Tokenize(text string) []string { return tokenize(text) }
func tokenize(text string) []string {
text = strings.ToLower(text)
text = strings.ReplaceAll(text, "-", " ")
text = strings.ReplaceAll(text, "_", " ")
return tokenRE.FindAllString(text, -1)
}
type document struct {
id int
title string
text string
source string
entry db.Entry
}
// TF-IDF index built from all observations
type TFIDFIndex struct {
docCount int
docFreq map[string]int // term → how many docs contain it
termDocs []map[string]float64 // per-doc: term → TF score
documents []document
}
func BuildTFIDF(entries []db.Entry) *TFIDFIndex {
idx := &TFIDFIndex{
docFreq: make(map[string]int),
termDocs: make([]map[string]float64, 0, len(entries)),
}
for _, e := range entries {
text := e.Title + " " + e.Summary + " " + e.SourceID + " " + e.Project
tokens := tokenize(text)
if len(tokens) == 0 {
continue
}
doc := document{
id: idx.docCount,
title: e.Title,
text: text,
source: e.SourceID,
entry: e,
}
// Term frequency in this doc
tf := make(map[string]float64)
seen := make(map[string]bool)
for _, t := range tokens {
if hexTokenRE.MatchString(t) || len(t) > 25 {
continue
}
tf[t]++
if !seen[t] {
idx.docFreq[t]++
seen[t] = true
}
}
// Normalize TF by doc length
docLen := float64(len(tokens))
for t, c := range tf {
tf[t] = c / docLen
}
idx.termDocs = append(idx.termDocs, tf)
idx.documents = append(idx.documents, doc)
idx.docCount++
}
return idx
}
func (idx *TFIDFIndex) TFIDF(term string, docIdx int) float64 {
tf, ok := idx.termDocs[docIdx][term]
if !ok {
return 0
}
df := idx.docFreq[term]
if df == 0 {
return 0
}
idf := math.Log(float64(idx.docCount) / float64(df))
return tf * idf
}
// TopTerms returns the N highest TF-IDF scoring terms for a document
func (idx *TFIDFIndex) TopTerms(docIdx int, n int) []struct {
Term string
Score float64
} {
var scored []struct {
Term string
Score float64
}
for t := range idx.termDocs[docIdx] {
s := idx.TFIDF(t, docIdx)
if s > 0 {
scored = append(scored, struct {
Term string
Score float64
}{t, s})
}
}
sort.Slice(scored, func(i, j int) bool { return scored[i].Score > scored[j].Score })
if len(scored) > n {
scored = scored[:n]
}
return scored
}
// Cluster groups observations into topics by shared top TF-IDF terms
func (idx *TFIDFIndex) Cluster(minSharedTerms int, maxTopics int) []TopicCluster {
if maxTopics <= 0 {
maxTopics = 20
}
if minSharedTerms <= 0 {
minSharedTerms = 2
}
// Get top 5 terms per doc
type docTerms struct {
docIdx int
terms []string
}
var docTermList []docTerms
for i := 0; i < idx.docCount; i++ {
tt := idx.TopTerms(i, 5)
if len(tt) >= minSharedTerms {
terms := make([]string, len(tt))
for j, t := range tt {
terms[j] = t.Term
}
docTermList = append(docTermList, docTerms{i, terms})
}
}
// Greedy clustering: docs sharing >= minSharedTerms terms become a topic
var clusters []TopicCluster
assigned := make(map[int]bool)
for _, dt := range docTermList {
if assigned[dt.docIdx] {
continue
}
cluster := TopicCluster{
Keywords: dt.terms,
Score: 0,
BySource: make(map[string]int),
}
// The seed doc belongs to this cluster too.
if seed := idx.documents[dt.docIdx]; seed.source != "" {
cluster.BySource[seed.source]++
cluster.Entries = append(cluster.Entries, seed.entry)
}
// Find all docs sharing terms with this seed
for _, other := range docTermList {
if assigned[other.docIdx] {
continue
}
shared := 0
for _, t1 := range dt.terms {
for _, t2 := range other.terms {
if t1 == t2 {
shared++
break
}
}
}
if shared >= minSharedTerms {
assigned[other.docIdx] = true
e := idx.documents[other.docIdx]
cluster.BySource[e.source]++
cluster.Entries = append(cluster.Entries, e.entry)
// Score is sum of TF-IDF of shared terms
for _, t := range dt.terms {
cluster.Score += idx.TFIDF(t, other.docIdx)
}
}
}
if len(cluster.BySource) > 0 {
// Name the topic by top scoring TF-IDF terms across all docs
allTerms := make(map[string]float64)
for _, t := range dt.terms {
for i := 0; i < idx.docCount; i++ {
allTerms[t] += idx.TFIDF(t, i)
}
}
type kv struct {
k string
v float64
}
var sorted []kv
for k, v := range allTerms {
sorted = append(sorted, kv{k, v})
}
sort.Slice(sorted, func(i, j int) bool { return sorted[i].v > sorted[j].v })
var nameParts []string
for _, kv := range sorted {
if len(nameParts) >= 3 {
break
}
nameParts = append(nameParts, kv.k)
}
cluster.Name = strings.Join(nameParts, " / ")
clusters = append(clusters, cluster)
if len(clusters) >= maxTopics {
break
}
}
}
return clusters
}
// Gaps finds topics with high browser activity but no skill coverage
func (idx *TFIDFIndex) Gaps(clusters []TopicCluster, skillKeywords map[string]bool) []TopicCluster {
var gaps []TopicCluster
for _, c := range clusters {
browserScore := c.BySource["browser-history"]
skillScore := c.BySource["skills-catalog"]
if browserScore > 0 && skillScore == 0 {
gaps = append(gaps, c)
}
}
sort.Slice(gaps, func(i, j int) bool {
return gaps[i].BySource["browser-history"] > gaps[j].BySource["browser-history"]
})
return gaps
}
+257
View File
@@ -0,0 +1,257 @@
package ingest
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"time"
_ "modernc.org/sqlite"
)
const (
chromiumHistoryPath = ".config/chromium/Default/History"
minHistoryPath = ".config/Min/BrowserHistory.db"
)
// noiseURLPatterns are URLs that represent navigation noise rather than knowledge.
var noiseURLPatterns = []string{
"opencode.ai",
"auth.opencode.ai",
"github.com/login/oauth/authorize",
"accounts.google.com",
"google.com/accounts",
"kagi.com",
"chatgpt.com",
// knox indexing its own web UI is self-referential noise
"localhost:8924",
"localhost:18925",
"127.0.0.1:8924",
"127.0.0.1:18925",
}
func noiseURL(url string) bool {
for _, p := range noiseURLPatterns {
if strings.Contains(url, p) {
return true
}
}
return false
}
type BrowserHistoryIngester struct {
Version string
}
func NewBrowserHistoryIngester() *BrowserHistoryIngester {
return &BrowserHistoryIngester{Version: "browser-history/v1"}
}
func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" }
type historyRow struct {
URL string
Title string
VisitTime time.Time
VisitCount int
}
func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) {
home, _ := os.UserHomeDir()
var results []*IngestResult
// Chromium
chromiumDB := filepath.Join(home, chromiumHistoryPath)
cr, err := b.ingestChromium(chromiumDB)
if err == nil {
results = append(results, cr...)
}
// Min
minDB := filepath.Join(home, minHistoryPath)
minResults, err := b.ingestMin(minDB)
if err == nil {
results = append(results, minResults...)
}
return results, nil
}
func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) {
if !fileExists(path) {
return nil, nil
}
// Copy to avoid lock issues (Chromium uses WAL)
tmp := path + ".knox_tmp"
if err := copyFile(path, tmp); err != nil {
return nil, fmt.Errorf("copy chromium db: %w", err)
}
defer os.Remove(tmp)
db, err := sql.Open("sqlite", tmp)
if err != nil {
return nil, fmt.Errorf("open chromium copy: %w", err)
}
defer db.Close()
rows, err := db.Query(
`SELECT url, COALESCE(title,''), last_visit_time, visit_count
FROM urls
WHERE last_visit_time > 0
ORDER BY last_visit_time DESC
LIMIT 1000`,
)
if err != nil {
// Table might not exist or schema mismatch
return nil, fmt.Errorf("query chromium urls: %w", err)
}
defer rows.Close()
var results []*IngestResult
for rows.Next() {
var url, title string
var visitTimeMicro int64
var visitCount int
if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil {
continue
}
// Chromium webkit time: microseconds since 1601-01-01 UTC
visitTime := webkitToTime(visitTimeMicro)
if result := b.makeResult(url, title, visitTime, visitCount); result != nil {
results = append(results, result)
}
}
return results, nil
}
func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) {
if !fileExists(path) {
return nil, nil
}
tmp := path + ".knox_tmp"
if err := copyFile(path, tmp); err != nil {
return nil, fmt.Errorf("copy min db: %w", err)
}
defer os.Remove(tmp)
db, err := sql.Open("sqlite", tmp)
if err != nil {
return nil, fmt.Errorf("open min copy: %w", err)
}
defer db.Close()
rows, err := db.Query(
`SELECT url, COALESCE(title,''), timestamp
FROM history
ORDER BY timestamp DESC
LIMIT 1000`,
)
if err != nil {
return nil, fmt.Errorf("query min history: %w", err)
}
defer rows.Close()
var results []*IngestResult
for rows.Next() {
var url, title string
var ts int64
if err := rows.Scan(&url, &title, &ts); err != nil {
continue
}
visitTime := time.Unix(ts, 0)
if result := b.makeResult(url, title, visitTime, 1); result != nil {
results = append(results, result)
}
}
return results, nil
}
func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult {
if url == "" {
return nil
}
// Skip chrome://, about://, file://, devtools
if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") ||
strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") ||
strings.HasPrefix(url, "chrome-extension://") {
return nil
}
if noiseURL(url) {
return nil
}
displayTitle := title
if displayTitle == "" {
displayTitle = extractDomain(url)
}
fp := Fingerprint([]byte(url))
confidence := 0.6
if visitCount > 5 {
confidence = 0.9
} else if visitCount > 1 {
confidence = 0.7
}
return &IngestResult{
Fingerprint: fp,
SourceID: b.SourceID(),
SourcePath: url,
ContentType: "url",
Title: displayTitle,
Summary: url,
CreatedAt: visitTime.Format(time.RFC3339),
Confidence: confidence,
IngesterVersion: b.Version,
Provenance: map[string]any{
"url": url,
"visit_count": visitCount,
"visited_at": visitTime.Format(time.RFC3339),
},
}
}
// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time
func webkitToTime(micros int64) time.Time {
// WebKit epoch: 1601-01-01 UTC
// Unix epoch: 1970-01-01 UTC
// Difference: 11644473600 seconds
secs := micros / 1_000_000
if secs < 11644473600 {
return time.Time{}
}
return time.Unix(secs-11644473600, 0)
}
func extractDomain(url string) string {
url = strings.TrimPrefix(url, "https://")
url = strings.TrimPrefix(url, "http://")
parts := strings.Split(url, "/")
if len(parts) > 0 {
return parts[0]
}
return url
}
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func copyFile(src, dst string) error {
data, err := os.ReadFile(src)
if err != nil {
return err
}
return os.WriteFile(dst, data, 0644)
}
+269
View File
@@ -0,0 +1,269 @@
package ingest
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
)
// GitIngester tracks project status for local git repositories.
//
// For each repo under one or more scan roots (default ~/src) it records the
// current branch, whether the working tree is clean, ahead/behind counts vs
// the upstream, the HEAD hash, and the last commit time + subject. The entry's
// CreatedAt is the last commit time, so staleness/dedup works naturally and
// an entry updates when new commits land.
//
// When Recursive is set, repos are discovered at any depth under a root
// (following symlinked directories), not just direct children — this picks up
// nested repos like ~/assistant/kitmaker or ~/Arduino/*/... Repos already have
// a stable identity via their real (symlink-resolved) path, so overlapping
// roots across symlinks do not duplicate entries.
type GitIngester struct {
Version string
Root string // single scan root; overridden by Roots (empty => $HOME/src)
Roots []string // multi-scan roots; overrides Root when non-empty
Recursive bool // recurse into subdirs (and follow dir symlinks) to find nested .git
}
func NewGitIngester() *GitIngester {
return &GitIngester{Version: "git/v1"}
}
func (g *GitIngester) SourceID() string { return "git" }
// scanRoots returns the effective list of scan roots.
func (g *GitIngester) scanRoots() []string {
if len(g.Roots) > 0 {
return g.Roots
}
if g.Root != "" {
return []string{g.Root}
}
home, err := os.UserHomeDir()
if err != nil {
return nil
}
return []string{filepath.Join(home, "src")}
}
func (g *GitIngester) IngestAll() ([]*IngestResult, error) {
// Dedup by real path so overlapping roots / symlinked roots don't double-add.
seen := make(map[string]struct{})
var results []*IngestResult
for _, root := range g.scanRoots() {
for _, repoPath := range collectGitRepos(root, g.Recursive) {
real, err := filepath.EvalSymlinks(repoPath)
if err != nil {
real = repoPath
}
if _, dup := seen[real]; dup {
continue
}
seen[real] = struct{}{}
if r := g.repoToResult(real); r != nil {
results = append(results, r)
}
}
}
return results, nil
}
func isGitRepo(path string) bool {
info, err := os.Stat(filepath.Join(path, ".git"))
return err == nil && info.IsDir()
}
// collectGitRepos returns paths of git repos under root. In non-recursive mode
// only direct children are considered (root itself is not counted), matching
// the original behavior. In recursive mode the root is walked to any depth,
// descending into symlinked directories, up to and including the root if it is
// itself a git repo. Descending stops once a git repo is found, so nested
// repos (e.g. a vendored copy inside another repo) are treated as leaves.
func collectGitRepos(root string, recursive bool) []string {
visited := make(map[string]struct{})
var out []string
var walk func(dir string)
walk = func(dir string) {
real, err := filepath.EvalSymlinks(dir)
if err != nil {
return
}
if _, ok := visited[real]; ok {
return
}
visited[real] = struct{}{}
if isGitRepo(real) {
out = append(out, real)
return
}
entries, err := os.ReadDir(real)
if err != nil {
return
}
for _, e := range entries {
if strings.HasPrefix(e.Name(), ".") {
continue
}
child := filepath.Join(real, e.Name())
if recursive {
walk(child)
continue
}
if e.IsDir() && isGitRepo(child) {
out = append(out, child)
}
}
}
walk(root)
return out
}
type GitStatus struct {
Branch string
Head string
Dirty int
Ahead int
Behind int
LastCommit string
Subject string
}
func (g *GitIngester) repoToResult(repoPath string) *IngestResult {
status := gitStatus(repoPath)
name := filepath.Base(repoPath)
fp := Fingerprint([]byte("git:repo:" + repoPath))
clean := "clean"
if status.Dirty > 0 {
clean = fmt.Sprintf("%d change(s)", status.Dirty)
}
summary := fmt.Sprintf("%s @ %s — %s", name, status.Branch, clean)
if status.Ahead > 0 || status.Behind > 0 {
summary += fmt.Sprintf(", ahead %d / behind %d", status.Ahead, status.Behind)
}
if status.Subject != "" {
summary += " — “" + truncate(status.Subject, 80) + "”"
}
createdAt := status.LastCommit
if createdAt == "" {
createdAt = gitStatusTime(status)
}
return &IngestResult{
Fingerprint: fp,
SourceID: g.SourceID(),
SourcePath: repoPath,
Project: name,
ContentType: "git-status",
Title: name,
Summary: summary,
CreatedAt: createdAt,
Confidence: 0.9,
IngesterVersion: g.Version,
Provenance: map[string]any{
"type": "git_status",
"path": repoPath,
"branch": status.Branch,
"head": status.Head,
"dirty": status.Dirty,
"ahead": status.Ahead,
"behind": status.Behind,
"last_commit": status.LastCommit,
"subject": status.Subject,
},
}
}
func gitStatusTime(s GitStatus) string {
if s.LastCommit != "" {
return s.LastCommit
}
return time.Now().UTC().Format(time.RFC3339)
}
func gitStatus(repoPath string) GitStatus {
out, err := gitOut(repoPath, "status", "--porcelain")
if err != nil {
return GitStatus{}
}
dirty := 0
if out != "" {
dirty = len(strings.Split(strings.TrimRight(out, "\n"), "\n"))
}
branch, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD")
head, _ := gitOut(repoPath, "rev-parse", "--short", "HEAD")
lastCommit, _ := gitOut(repoPath, "log", "-1", "--format=%cI")
subject, _ := gitOut(repoPath, "log", "-1", "--format=%s")
ahead, behind := 0, 0
// ahead/behind only meaningful if there's an upstream
if upOut, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD@{upstream}"); upOut != "" {
if rev, err := gitOut(repoPath, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err == nil {
fields := strings.Fields(rev)
if len(fields) == 2 {
var a, b int
if _, e1 := fmt.Sscanf(fields[0], "%d", &a); e1 == nil {
ahead = a
}
if _, e2 := fmt.Sscanf(fields[1], "%d", &b); e2 == nil {
behind = b
}
}
}
}
return GitStatus{
Branch: strings.TrimSpace(branch),
Head: strings.TrimSpace(head),
Dirty: dirty,
Ahead: ahead,
Behind: behind,
LastCommit: strings.TrimSpace(lastCommit),
Subject: strings.TrimSpace(subject),
}
}
// sanitizeGitField strips ANSI codes and control bytes from git output
// (colour/unicode modes can inject escape sequences).
var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
func sanitizeGitField(s string) string {
s = ansiRe.ReplaceAllString(s, "")
s = strings.ReplaceAll(s, "\x1b", "")
return strings.TrimSpace(s)
}
// gitOut runs git in repoPath and returns trimmed stdout (or "" on error).
func gitOut(repoPath string, args ...string) (string, error) {
cmd := exec.Command("git", append([]string{"-C", repoPath}, args...)...)
cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1")
out, err := cmd.Output()
if err != nil {
return "", err
}
return sanitizeGitField(string(out)), nil
}
func (g *GitIngester) Ingest(_ string) (*IngestResult, error) {
return nil, fmt.Errorf("use IngestAll() for git")
}
var _ Ingester = (*GitIngester)(nil)
// SortResults orders results deterministically by project name (helper for CLI output).
func SortResults(res []*IngestResult) {
sort.Slice(res, func(i, j int) bool { return res[i].Project < res[j].Project })
}
+245
View File
@@ -0,0 +1,245 @@
package ingest
import (
"encoding/json"
"fmt"
"os/exec"
"time"
)
type GiteaIngester struct {
Version string
}
func NewGiteaIngester() *GiteaIngester {
return &GiteaIngester{Version: "gitea-tea/v1"}
}
func (g *GiteaIngester) SourceID() string { return "gitea" }
func (g *GiteaIngester) IngestAll() ([]*IngestResult, error) {
var results []*IngestResult
repos, err := g.fetchRepos()
if err == nil {
for _, r := range repos {
if result := g.repoToResult(r); result != nil {
results = append(results, result)
}
}
}
issues, err := g.fetchIssues()
if err == nil {
for _, i := range issues {
if result := g.issueToResult(i); result != nil {
results = append(results, result)
}
}
}
pulls, err := g.fetchPulls()
if err == nil {
for _, p := range pulls {
if result := g.pullToResult(p); result != nil {
results = append(results, result)
}
}
}
return results, nil
}
type teaRepo struct {
Owner string `json:"owner"`
Name string `json:"name"`
Description string `json:"description"`
Updated string `json:"updated"`
}
type teaIssue struct {
Index string `json:"index"`
Title string `json:"title"`
State string `json:"state"`
Owner string `json:"owner"`
Repo string `json:"repo"`
Labels string `json:"labels"`
Author string `json:"author"`
Updated string `json:"updated"`
}
type teaPull struct {
Index string `json:"index"`
Title string `json:"title"`
State string `json:"state"`
Owner string `json:"owner"`
Repo string `json:"repo"`
Updated string `json:"updated"`
}
func (g *GiteaIngester) teaJSON(args ...string) ([][]byte, error) {
cmd := exec.Command("tea", append(args, "--output", "json")...)
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("tea %s: %w", args[0], err)
}
var raw json.RawMessage
if err := json.Unmarshal(output, &raw); err != nil {
return nil, err
}
// Handle both array and object responses
var arr []json.RawMessage
if err := json.Unmarshal(raw, &arr); err == nil {
bytes := make([][]byte, len(arr))
for i, a := range arr {
bytes[i] = a
}
return bytes, nil
}
// Single object — wrap in array
return [][]byte{raw}, nil
}
func (g *GiteaIngester) fetchRepos() ([]teaRepo, error) {
items, err := g.teaJSON("repos", "list", "--limit", "100", "--fields", "owner,name,description,updated")
if err != nil {
return nil, err
}
var repos []teaRepo
for _, item := range items {
var r teaRepo
if err := json.Unmarshal(item, &r); err == nil {
repos = append(repos, r)
}
}
return repos, nil
}
func (g *GiteaIngester) fetchIssues() ([]teaIssue, error) {
items, err := g.teaJSON("issues", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,author,milestone,labels,owner,repo,updated")
if err != nil {
return nil, err
}
var issues []teaIssue
for _, item := range items {
var i teaIssue
if err := json.Unmarshal(item, &i); err == nil {
issues = append(issues, i)
}
}
return issues, nil
}
func (g *GiteaIngester) fetchPulls() ([]teaPull, error) {
items, err := g.teaJSON("pulls", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,owner,repo,updated")
if err != nil {
return nil, err
}
var pulls []teaPull
for _, item := range items {
var p teaPull
if err := json.Unmarshal(item, &p); err == nil {
pulls = append(pulls, p)
}
}
return pulls, nil
}
// signalTime returns the item's last-activity time, falling back to now.
func signalTime(updated string) string {
if updated != "" {
if _, err := time.Parse(time.RFC3339, updated); err == nil {
return updated
}
}
return time.Now().UTC().Format(time.RFC3339)
}
func (g *GiteaIngester) repoToResult(r teaRepo) *IngestResult {
fullName := r.Owner + "/" + r.Name
fp := Fingerprint([]byte("gitea:repo:" + fullName))
return &IngestResult{
Fingerprint: fp,
SourceID: g.SourceID(),
SourcePath: fullName,
Project: r.Name,
ContentType: "repo",
Title: fullName,
Summary: r.Description,
Confidence: 0.9,
IngesterVersion: g.Version,
CreatedAt: signalTime(r.Updated),
Provenance: map[string]any{
"type": "repository",
"owner": r.Owner,
"name": r.Name,
"description": r.Description,
},
}
}
func (g *GiteaIngester) issueToResult(i teaIssue) *IngestResult {
title := fmt.Sprintf("#%s %s", i.Index, i.Title)
fp := Fingerprint([]byte(fmt.Sprintf("gitea:issue:%s/%s:%s", i.Owner, i.Repo, i.Index)))
summary := fmt.Sprintf("[%s/%s#%s] %s", i.Owner, i.Repo, i.Index, i.Title)
if i.Labels != "" {
summary += " [" + i.Labels + "]"
}
return &IngestResult{
Fingerprint: fp,
SourceID: g.SourceID(),
SourcePath: fmt.Sprintf("%s/%s#%s", i.Owner, i.Repo, i.Index),
Project: i.Repo,
ContentType: "issue",
Title: title,
Summary: summary,
Confidence: 0.85,
IngesterVersion: g.Version,
CreatedAt: signalTime(i.Updated),
Provenance: map[string]any{
"type": "issue",
"owner": i.Owner,
"repo": i.Repo,
"index": i.Index,
"state": i.State,
"labels": i.Labels,
},
}
}
func (g *GiteaIngester) pullToResult(p teaPull) *IngestResult {
title := fmt.Sprintf("!%s %s", p.Index, p.Title)
fp := Fingerprint([]byte(fmt.Sprintf("gitea:pull:%s/%s:%s", p.Owner, p.Repo, p.Index)))
return &IngestResult{
Fingerprint: fp,
SourceID: g.SourceID(),
SourcePath: fmt.Sprintf("%s/%s!%s", p.Owner, p.Repo, p.Index),
Project: p.Repo,
ContentType: "pull",
Title: title,
Summary: fmt.Sprintf("[PR %s/%s] %s", p.Owner, p.Repo, p.Title),
Confidence: 0.85,
IngesterVersion: g.Version,
CreatedAt: signalTime(p.Updated),
Provenance: map[string]any{
"type": "pull_request",
"owner": p.Owner,
"repo": p.Repo,
"index": p.Index,
"state": p.State,
},
}
}
func (g *GiteaIngester) Ingest(_ string) (*IngestResult, error) {
return nil, fmt.Errorf("use IngestAll() for gitea")
}
var _ Ingester = (*GiteaIngester)(nil)
+57
View File
@@ -0,0 +1,57 @@
package ingest
import (
"crypto/sha256"
"encoding/json"
"fmt"
)
type IngestResult struct {
Fingerprint string
SourceID string
SourcePath string
Project string
ContentType string
Title string
Summary string
CreatedAt string
LineStart int
LineEnd int
Confidence float64
IngesterVersion string
Provenance map[string]any
}
type Ingester interface {
SourceID() string
Ingest(path string) (*IngestResult, error)
}
func Fingerprint(data []byte) string {
h := sha256.Sum256(data)
return fmt.Sprintf("%x", h[:16])
}
func FingerprintWithMeta(data []byte, meta map[string]string) string {
h := sha256.New()
h.Write(data)
enc := json.NewEncoder(h)
_ = enc.Encode(meta)
return fmt.Sprintf("%x", h.Sum(nil)[:16])
}
func ProvenanceJSON(m map[string]any) string {
b, err := json.Marshal(m)
if err != nil {
return "{}"
}
return string(b)
}
func truncate(s string, n int) string {
runes := []rune(s)
if len(runes) <= n {
return s
}
return string(runes[:n]) + "..."
}
+138
View File
@@ -0,0 +1,138 @@
package ingest
import (
"bufio"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
var _ Ingester = (*LogIngester)(nil)
type LogIngester struct {
Version string
}
func NewLogIngester() *LogIngester {
return &LogIngester{Version: "log-parser/v1"}
}
func (l *LogIngester) SourceID() string { return "opencode-log" }
var (
sessionIDRE = regexp.MustCompile(`session[=_ ]?([a-zA-Z0-9_-]+)`)
projectIDRE = regexp.MustCompile(`project[=_ ]?([a-zA-Z0-9_.-]+)`)
modelRE = regexp.MustCompile(`model[=_ ]?([a-zA-Z0-9_.-]+)`)
agentRE = regexp.MustCompile(`agent[=_ ]?([a-zA-Z0-9_.-]+)`)
errorRE = regexp.MustCompile(`(?i)(error|fail|exception|panic|timeout)`)
)
func (l *LogIngester) Ingest(path string) (*IngestResult, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open log: %w", err)
}
defer f.Close()
var (
sessionIDs []string
models []string
agents []string
errors []string
totalLines int
firstError int
)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
totalLines++
if m := sessionIDRE.FindStringSubmatch(line); len(m) > 1 {
sessionIDs = append(sessionIDs, m[1])
}
if m := modelRE.FindStringSubmatch(line); len(m) > 1 {
models = append(models, m[1])
}
if m := agentRE.FindStringSubmatch(line); len(m) > 1 {
agents = append(agents, m[1])
}
if errorRE.MatchString(line) {
if len(errors) == 0 {
firstError = totalLines
}
errors = append(errors, truncate(line, 120))
}
}
base := filepath.Base(path)
fp := Fingerprint([]byte(path))
title := fmt.Sprintf("Log %s (%d lines)", base, totalLines)
createdAt := ""
if fi, err := os.Stat(path); err == nil {
createdAt = fi.ModTime().UTC().Format(time.RFC3339)
}
var summaryParts []string
if len(sessionIDs) > 0 {
summaryParts = append(summaryParts, fmt.Sprintf("sessions: %s", uniqueJoin(sessionIDs, 5)))
}
if len(models) > 0 {
summaryParts = append(summaryParts, fmt.Sprintf("models: %s", uniqueJoin(models, 3)))
}
if len(errors) > 0 {
summaryParts = append(summaryParts, fmt.Sprintf("%d errors (line %d)", len(errors), firstError))
}
summary := strings.Join(summaryParts, " | ")
if summary == "" {
summary = fmt.Sprintf("%d lines", totalLines)
}
confidence := 0.6
if len(errors) > 0 {
confidence = 0.8
}
return &IngestResult{
Fingerprint: fp,
SourceID: l.SourceID(),
SourcePath: path,
ContentType: ".log",
Title: title,
Summary: summary,
CreatedAt: createdAt,
Confidence: confidence,
IngesterVersion: l.Version,
LineEnd: totalLines,
Provenance: map[string]any{
"session_ids": uniqueSlice(sessionIDs),
"models": uniqueSlice(models),
"agents": uniqueSlice(agents),
"error_count": len(errors),
"file_size": fmt.Sprintf("%d lines", totalLines),
},
}, nil
}
func uniqueJoin(items []string, max int) string {
uniq := uniqueSlice(items)
if len(uniq) > max {
uniq = uniq[:max]
}
return strings.Join(uniq, ", ")
}
func uniqueSlice(items []string) []string {
seen := make(map[string]bool)
var uniq []string
for _, s := range items {
if !seen[s] {
seen[s] = true
uniq = append(uniq, s)
}
}
return uniq
}
+267
View File
@@ -0,0 +1,267 @@
package ingest
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
const obsidianConfigPath = ".config/obsidian/obsidian.json"
type ObsidianIngester struct {
Version string
Vault string // optional override; empty = auto-detect
}
func NewObsidianIngester(vault string) *ObsidianIngester {
return &ObsidianIngester{Version: "obsidian-note/v1", Vault: vault}
}
func (o *ObsidianIngester) SourceID() string { return "obsidian" }
func (o *ObsidianIngester) IngestAll() ([]*IngestResult, error) {
vault := o.Vault
if vault == "" {
var err error
vault, err = detectObsidianVault()
if err != nil {
return nil, fmt.Errorf("no vault path and auto-detect failed: %w", err)
}
}
if !fileExists(vault) {
return nil, fmt.Errorf("vault path %s does not exist", vault)
}
var results []*IngestResult
err := filepath.Walk(vault, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // skip errors per-file
}
if info.IsDir() {
dir := filepath.Base(path)
if strings.HasPrefix(dir, ".") || dir == "node_modules" {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(path, ".md") {
return nil
}
result, err := o.ingestNote(path, vault)
if err != nil {
return nil
}
if result != nil {
results = append(results, result)
}
return nil
})
return results, err
}
func (o *ObsidianIngester) ingestNote(path, vault string) (*IngestResult, error) {
data, err := os.ReadFile(path)
if err != nil || len(data) < 10 {
return nil, nil
}
content := string(data)
title := extractObsidianTitle(content, path, vault)
tags := extractObsidianTags(content)
created, modified := extractObsidianTimestamps(content, path)
if created == "" {
if fi, err := os.Stat(path); err == nil {
created = fi.ModTime().UTC().Format(time.RFC3339)
}
}
if modified == "" && created != "" {
modified = created
}
// Fingerprint by content (so edits create updates)
fp := FingerprintWithMeta(data, map[string]string{
"source": "obsidian",
"path": path,
})
relPath, _ := filepath.Rel(vault, path)
summary := extractBodyPreview(content, 200)
return &IngestResult{
Fingerprint: fp,
SourceID: o.SourceID(),
SourcePath: relPath,
Project: filepath.Base(vault),
ContentType: ".md",
Title: title,
Summary: summary,
CreatedAt: created,
LineStart: 0,
LineEnd: len(strings.Split(content, "\n")),
Confidence: 0.85,
IngesterVersion: o.Version,
Provenance: map[string]any{
"path": relPath,
"tags": tags,
"modified_at": modified,
"note_title": title,
},
}, nil
}
func DetectObsidianVault() (string, error) { return detectObsidianVault() }
func detectObsidianVault() (string, error) {
home, _ := os.UserHomeDir()
cfgPath := filepath.Join(home, obsidianConfigPath)
if !fileExists(cfgPath) {
// Fallback: check common vault locations
common := []string{
filepath.Join(home, "notes"),
filepath.Join(home, "Documents", "notes"),
filepath.Join(home, "Obsidian"),
filepath.Join(home, "vault"),
}
for _, p := range common {
if fileExists(filepath.Join(p, ".obsidian")) {
return p, nil
}
}
return "", fmt.Errorf("no obsidian config found at %s and no common vault detected", cfgPath)
}
data, err := os.ReadFile(cfgPath)
if err != nil {
return "", err
}
var cfg struct {
Vaults map[string]struct {
Path string `json:"path"`
} `json:"vaults"`
}
if err := json.Unmarshal(data, &cfg); err != nil {
return "", fmt.Errorf("parse obsidian config: %w", err)
}
for _, v := range cfg.Vaults {
if v.Path != "" {
return v.Path, nil
}
}
return "", fmt.Errorf("no vault path in obsidian config")
}
func extractObsidianTitle(content, path, vault string) string {
// Try frontmatter title first
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "title:") {
t := strings.TrimSpace(trimmed[6:])
t = strings.Trim(t, "\"'")
if t != "" {
return t
}
}
if strings.HasPrefix(trimmed, "aliases:") {
break
}
if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
break
}
}
// Fallback: first H1
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "# ") {
return strings.TrimSpace(trimmed[2:])
}
}
// Fallback: filename
rel, _ := filepath.Rel(vault, path)
return strings.TrimSuffix(rel, ".md")
}
func extractObsidianTags(content string) []string {
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "tags:") {
raw := strings.TrimSpace(trimmed[5:])
raw = strings.Trim(raw, "\"[] ")
var tags []string
for _, t := range strings.Split(raw, ",") {
t = strings.TrimSpace(t)
t = strings.Trim(t, "\" ")
if t != "" {
tags = append(tags, t)
}
}
return tags
}
if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
break
}
}
return nil
}
func extractObsidianTimestamps(content, path string) (created, modified string) {
// Try frontmatter dates
inFrontmatter := false
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "---" {
inFrontmatter = !inFrontmatter
continue
}
if !inFrontmatter {
break
}
if strings.HasPrefix(trimmed, "created:") || strings.HasPrefix(trimmed, "date:") {
created = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
created = strings.Trim(created, "\"' ")
}
if strings.HasPrefix(trimmed, "modified:") || strings.HasPrefix(trimmed, "updated:") {
modified = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
modified = strings.Trim(modified, "\"' ")
}
}
return created, modified
}
func extractBodyPreview(content string, maxLen int) string {
inFrontmatter := false
var body strings.Builder
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "---" {
if inFrontmatter {
inFrontmatter = false
continue
}
inFrontmatter = true
continue
}
if inFrontmatter {
continue
}
if trimmed != "" && body.Len() < maxLen {
if body.Len() > 0 {
body.WriteString(" ")
}
body.WriteString(truncate(trimmed, maxLen-body.Len()))
}
}
result := body.String()
if len(result) > maxLen {
return result[:maxLen] + "..."
}
return result
}
+69
View File
@@ -0,0 +1,69 @@
package ingest
import (
"os"
"path/filepath"
"strings"
"time"
)
var _ Ingester = (*ObsidianFileIngester)(nil)
type ObsidianFileIngester struct {
Version string
Vault string
}
func NewObsidianFileIngester(vault string) *ObsidianFileIngester {
return &ObsidianFileIngester{Version: "obsidian-file/v1", Vault: vault}
}
func (o *ObsidianFileIngester) SourceID() string { return "obsidian" }
func (o *ObsidianFileIngester) Ingest(path string) (*IngestResult, error) {
if !strings.HasSuffix(path, ".md") {
return nil, nil
}
data, err := os.ReadFile(path)
if err != nil || len(data) < 10 {
return nil, nil
}
content := string(data)
title := extractObsidianTitle(content, path, o.Vault)
tags := extractObsidianTags(content)
created, _ := extractObsidianTimestamps(content, path)
if created == "" {
if fi, err := os.Stat(path); err == nil {
created = fi.ModTime().UTC().Format(time.RFC3339)
}
}
fp := FingerprintWithMeta(data, map[string]string{
"source": "obsidian",
"path": path,
})
relPath, _ := filepath.Rel(o.Vault, path)
summary := extractBodyPreview(content, 200)
return &IngestResult{
Fingerprint: fp,
SourceID: o.SourceID(),
SourcePath: relPath,
Project: filepath.Base(o.Vault),
ContentType: ".md",
Title: title,
Summary: summary,
CreatedAt: created,
LineEnd: len(strings.Split(content, "\n")),
Confidence: 0.85,
IngesterVersion: o.Version,
Provenance: map[string]any{
"path": relPath,
"tags": tags,
"note_title": title,
},
}, nil
}
+157
View File
@@ -0,0 +1,157 @@
package ingest
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
type jsonKind int
const (
jsonObject jsonKind = iota
jsonArray
)
var _ Ingester = (*SessionDiffIngester)(nil)
type SessionDiffIngester struct {
Version string
}
func NewSessionDiffIngester() *SessionDiffIngester {
return &SessionDiffIngester{Version: "session-diff/v1"}
}
func (s *SessionDiffIngester) SourceID() string { return "opencode-session" }
func (s *SessionDiffIngester) Ingest(path string) (*IngestResult, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read session diff: %w", err)
}
if len(data) < 10 {
return nil, nil
}
raw, kind := parseJSON(data)
if raw == nil {
return nil, nil
}
sessionID := guessString(raw, "session_id", "id", "sessionId")
project := guessString(raw, "project", "project_id", "projectPath", "projectId")
title := guessString(raw, "title", "name", "label")
createdAt := guessString(raw, "created_at", "timestamp", "date", "started_at")
if createdAt == "" {
if fi, err := os.Stat(path); err == nil {
createdAt = fi.ModTime().UTC().Format(time.RFC3339)
}
}
if sessionID == "" {
base := filepath.Base(path)
sessionID = strings.TrimSuffix(base, filepath.Ext(base))
sessionID = strings.TrimPrefix(sessionID, "ses_")
}
if title == "" {
title = fmt.Sprintf("Session %s", sessionID[:min(8, len(sessionID))])
}
summary := extractSummary(raw, kind)
fp := FingerprintWithMeta(data, map[string]string{
"session_id": sessionID,
"source": "session_diff",
})
confidence := 0.7
if kind == jsonArray {
confidence = 0.4
}
return &IngestResult{
Fingerprint: fp,
SourceID: s.SourceID(),
SourcePath: path,
Project: project,
ContentType: ".json",
Title: title,
Summary: summary,
CreatedAt: createdAt,
LineStart: 0,
LineEnd: 0,
Confidence: confidence,
IngesterVersion: s.Version,
Provenance: map[string]any{
"session_id": sessionID,
"kind": kindName(kind),
"field_count": len(raw),
},
}, nil
}
func parseJSON(data []byte) (map[string]any, jsonKind) {
var obj map[string]any
if err := json.Unmarshal(data, &obj); err == nil {
return obj, jsonObject
}
var arr []any
if err := json.Unmarshal(data, &arr); err == nil {
m := make(map[string]any)
m["_count"] = float64(len(arr))
m["_kind"] = "array"
if len(arr) > 0 {
if first, ok := arr[0].(map[string]any); ok {
for k, v := range first {
if str, ok := v.(string); ok && len(str) < 200 {
m["_sample_"+k] = str
break
}
}
}
}
return m, jsonArray
}
return nil, jsonObject
}
func extractSummary(raw map[string]any, kind jsonKind) string {
if msg, ok := raw["message"].(string); ok && len(msg) > 0 {
return truncate(msg, 200)
}
if msgs, ok := raw["messages"].([]any); ok && len(msgs) > 0 {
parts := make([]string, 0, len(msgs))
for _, m := range msgs {
if mm, ok := m.(map[string]any); ok {
if c, ok := mm["content"].(string); ok {
parts = append(parts, truncate(c, 100))
}
}
}
return strings.Join(parts, " | ")
}
if count, ok := raw["_count"].(float64); ok && kind == jsonArray {
return fmt.Sprintf("Array with %.0f entries", count)
}
b, _ := json.Marshal(raw)
return truncate(string(b), 200)
}
func guessString(m map[string]any, keys ...string) string {
for _, k := range keys {
if v, ok := m[k].(string); ok && v != "" {
return v
}
}
return ""
}
func kindName(k jsonKind) string {
if k == jsonArray {
return "array"
}
return "object"
}
+135
View File
@@ -0,0 +1,135 @@
package ingest
import (
"fmt"
"os"
"path/filepath"
"strings"
)
var _ Ingester = (*SkillsIngester)(nil)
type SkillsIngester struct {
Version string
}
func NewSkillsIngester() *SkillsIngester {
return &SkillsIngester{Version: "skills-index/v1"}
}
func (s *SkillsIngester) SourceID() string { return "skills-catalog" }
func (s *SkillsIngester) Ingest(path string) (*IngestResult, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read skill: %w", err)
}
if len(data) < 20 {
return nil, nil
}
content := string(data)
title := extractSkillName(content, path)
summary := extractSkillSummary(content)
tags := extractSkillTags(content)
projects := extractProjects(tags)
createdAt := extractCreatedAt(content)
fp := FingerprintWithMeta(data, map[string]string{
"source": "skills-catalog",
"title": title,
})
return &IngestResult{
Fingerprint: fp,
SourceID: s.SourceID(),
SourcePath: path,
Project: projects,
ContentType: ".md",
Title: title,
Summary: summary,
CreatedAt: createdAt,
LineStart: 0,
LineEnd: len(strings.Split(content, "\n")),
Confidence: 0.9,
IngesterVersion: s.Version,
Provenance: map[string]any{
"skill_name": title,
"tags": tags,
"line_count": len(strings.Split(content, "\n")),
},
}, nil
}
func extractSkillName(content, path string) string {
base := filepath.Base(filepath.Dir(path))
if base != "" && base != "." && base != "/" {
return base
}
return strings.TrimSuffix(filepath.Base(path), ".md")
}
func extractSkillSummary(content string) string {
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "description:") {
desc := strings.TrimSpace(trimmed[len("description:"):])
if len(desc) > 200 {
desc = desc[:200] + "..."
}
return desc
}
}
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "# ") {
continue
}
if trimmed != "" && !strings.HasPrefix(trimmed, "---") {
return truncate(trimmed, 200)
}
}
return ""
}
func extractSkillTags(content string) []string {
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "tags:") {
raw := strings.TrimSpace(trimmed[len("tags:"):])
raw = strings.Trim(raw, "\"[]")
var tags []string
for _, t := range strings.Split(raw, ",") {
t = strings.TrimSpace(t)
t = strings.Trim(t, "\" ")
if t != "" {
tags = append(tags, t)
}
}
return tags
}
}
return nil
}
func extractProjects(tags []string) string {
for _, t := range tags {
if strings.HasPrefix(t, "scope:project:") {
return strings.TrimPrefix(t, "scope:project:")
}
if strings.HasPrefix(t, "scope:machine:") {
return "[machine]"
}
}
return ""
}
func extractCreatedAt(content string) string {
for _, line := range strings.Split(content, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "created_at:") || strings.HasPrefix(trimmed, "updated_at:") {
return strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
}
}
return ""
}
+299
View File
@@ -0,0 +1,299 @@
{{define "head"}}<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Knox{{if .title}} · {{.title}}{{end}}</title>
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
<script src="https://cdn.tailwindcss.com"></script>
<style>body{font-family:system-ui,sans-serif}</style>
</head>
<body class="bg-gray-950 text-gray-100 min-h-screen">
<header class="border-b border-gray-800 p-4 flex items-center justify-between">
<h1 class="text-xl font-bold text-emerald-400"><a href="/">Knox</a>{{if .title}} <span class="text-gray-400 text-sm font-normal">/ {{.title}}</span>{{end}}</h1>
<nav class="flex gap-4 text-sm">
<a href="/" class="text-gray-300 hover:text-white">Dashboard</a>
<a href="/threads" class="text-gray-300 hover:text-white">Threads</a>
</nav>
</header>
{{end}}
{{define "foot"}}</body>
</html>
{{end}}
{{define "not_found"}}
{{template "head" .}}
<main class="max-w-4xl mx-auto p-6 text-center">
<div class="text-gray-500 text-lg">Page not found</div>
<a href="/" class="mt-6 inline-block text-emerald-400 hover:text-emerald-300 text-sm">Back to Dashboard</a>
</main>
{{template "foot" .}}
{{end}}
{{define "dashboard"}}
{{template "head" .}}
<main class="max-w-6xl mx-auto p-6 space-y-6">
{{if .golden}}
<div class="bg-emerald-900/30 border border-emerald-800 rounded-lg p-4">
<div class="text-xs text-emerald-400 uppercase tracking-wider">Golden Thread</div>
<div class="text-lg font-semibold mt-1">{{.golden.Title}}</div>
<div class="text-sm text-gray-400">{{.golden.Motivation}}</div>
</div>
{{end}}
<div class="flex gap-4">
<input type="text" name="q" placeholder="Search knowledge index..."
class="flex-1 bg-gray-900 border border-gray-700 rounded px-4 py-2 text-sm focus:outline-none focus:border-emerald-500"
hx-get="/search" hx-trigger="keyup changed delay:300ms" hx-target="#results" hx-indicator="#search-indicator">
<button class="bg-emerald-700 hover:bg-emerald-600 px-4 py-2 rounded text-sm font-medium"
hx-get="/search" hx-include="[name=q]" hx-target="#results" hx-indicator="#search-indicator">Search</button>
</div>
<div id="results" class="text-sm text-gray-500"></div>
<div id="search-indicator" class="htmx-indicator text-xs text-emerald-400">Searching...</div>
<div hx-get="/stats" hx-trigger="load, every 30s" hx-swap="innerHTML">
<div class="text-gray-500">Loading stats...</div>
</div>
<section>
<h2 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Issues</h2>
{{if .gitea}}
<div class="space-y-1 mb-4">
{{range .gitea}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<span class="text-emerald-300">{{trunc .Title 80}}</span>
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
</a>
{{end}}
</div>
{{end}}
</section>
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
<section>
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Notes</h3>
{{if .obsidian}}
<div class="space-y-1">
{{range .obsidian}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<span>{{trunc .Title 50}}</span>
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
</a>
{{end}}
</div>
{{else}}
<div class="text-gray-500 text-xs">No notes indexed.</div>
{{end}}
</section>
<section>
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Sessions</h3>
{{if .session}}
<div class="space-y-1">
{{range .session}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<span>{{trunc .Title 50}}</span>
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
</a>
{{end}}
</div>
{{else}}
<div class="text-gray-500 text-xs">No sessions tracked.</div>
{{end}}
</section>
<section>
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Skills</h3>
{{if .skills}}
<div class="space-y-1">
{{range .skills}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<span>{{trunc .Title 50}}</span>
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
</a>
{{end}}
</div>
{{else}}
<div class="text-gray-500 text-xs">No skills indexed.</div>
{{end}}
</section>
</div>
<section>
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Feed</h3>
<div class="space-y-1">
{{range .recent}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<span class="font-mono text-xs text-gray-500">{{shortFP .Fingerprint}}</span>
<span class="ml-2">{{trunc .Title 60}}</span>
<span class="ml-2 text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.SourceID}}</span>
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
</a>
{{else}}
<div class="text-gray-500 text-sm">No entries yet.</div>
{{end}}
</div>
</section>
</main>
{{template "foot" .}}
{{end}}
{{define "search_results"}}
{{if .results}}
<div class="space-y-1">
<div class="text-xs text-gray-500 mb-2">{{len .results}} results for "{{.query}}"</div>
{{range .results}}
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
<div>
<span class="font-mono text-xs text-gray-500">{{shortFP .Fingerprint}}</span>
<span class="ml-2 font-medium">{{trunc .Title 70}}</span>
<span class="ml-2 text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.SourceID}}</span>
</div>
{{if .Summary}}<div class="text-gray-500 text-xs mt-0.5 ml-12">{{trunc .Summary 120}}</div>{{end}}
</a>
{{end}}
</div>
{{else}}
<p class="text-gray-500 text-sm">No results for "{{.query}}".</p>
{{end}}
{{end}}
{{define "stats"}}
<div class="grid grid-cols-2 md:grid-cols-4 gap-4">
{{range .stats}}
<div class="bg-gray-900 rounded-lg p-3 border border-gray-800">
<div class="text-xs text-gray-500 uppercase tracking-wider">{{.Label}}</div>
<div class="text-xl font-semibold mt-1">{{.Value}}</div>
</div>
{{end}}
</div>
{{end}}
{{define "thread_list"}}
{{template "head" .}}
<main class="max-w-4xl mx-auto p-6 space-y-4">
<div class="flex gap-2 mb-4">
<a href="/threads" class="px-3 py-1 text-sm rounded {{if eq .status ""}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">All</a>
<a href="/threads?status=active" class="px-3 py-1 text-sm rounded {{if eq .status "active"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Active</a>
<a href="/threads?status=stalled" class="px-3 py-1 text-sm rounded {{if eq .status "stalled"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Stalled</a>
<a href="/threads?status=resolved" class="px-3 py-1 text-sm rounded {{if eq .status "resolved"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Resolved</a>
</div>
{{range .threads}}
<a href="/threads/{{.ID}}" class="block bg-gray-900 rounded-lg p-4 border border-gray-800 hover:border-gray-600">
<div class="flex items-center gap-2">
<span class="text-lg font-semibold">{{.Title}}</span>
<span class="text-xs px-1.5 py-0.5 rounded {{if eq .Status "active"}}bg-emerald-900 text-emerald-300{{else if eq .Status "resolved"}}bg-blue-900 text-blue-300{{else}}bg-gray-800 text-gray-400{{end}}">{{.Status}}</span>
<span class="text-xs text-gray-500">{{.Priority}}</span>
</div>
{{if .Motivation}}<div class="text-sm text-gray-400 mt-1">{{.Motivation}}</div>{{end}}
<div class="text-xs text-gray-600 mt-1">{{.EntryCount}} observations · {{humanTime .UpdatedAt}}</div>
</a>
{{else}}
<div class="text-gray-500 text-sm">No threads found.</div>
{{end}}
</main>
{{template "foot" .}}
{{end}}
{{define "thread_detail"}}
{{template "head" .}}
<main class="max-w-4xl mx-auto p-6 space-y-6">
<div class="bg-gray-900 rounded-lg p-4 border border-gray-800">
<div class="flex items-center gap-2">
<h2 class="text-xl font-bold">{{.thread.Title}}</h2>
{{if .isGolden}}<span class="text-xs bg-emerald-900 text-emerald-300 px-2 py-0.5 rounded">GOLDEN</span>{{end}}
</div>
<div class="flex gap-4 mt-2 text-sm text-gray-400">
<span>{{.thread.Status}}</span>
<span>{{.thread.Priority}}</span>
<span>Created {{humanTime .thread.CreatedAt}}</span>
</div>
{{if .thread.Motivation}}<div class="mt-3 text-gray-300"><strong>Motivation:</strong> {{.thread.Motivation}}</div>{{end}}
{{if ne .thread.Provenance "{}"}}<div class="mt-1 text-xs text-gray-500"><strong>Provenance:</strong> {{.thread.Provenance}}</div>{{end}}
</div>
{{if .provenance}}
<section>
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Provenance Graph</h3>
<div class="bg-gray-900 rounded-lg p-3 border border-gray-800 text-sm font-mono">
{{range .provenance}}
<div class="text-gray-400">└─ <a href="{{fpURL .Fingerprint}}" class="text-emerald-400 hover:text-emerald-300">{{shortFP .Fingerprint}}</a> <span class="text-gray-600">[{{.Relation}}]</span></div>
{{end}}
</div>
</section>
{{end}}
{{if .obs}}
<section>
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Observations ({{len .obs}})</h3>
<div class="space-y-1">
{{range .obs}}
<div class="bg-gray-900 rounded px-3 py-2 text-sm flex items-center gap-3">
<span class="font-mono text-xs text-gray-500">#{{.ID}}</span>
<span class="flex-1">{{trunc .Title 50}}</span>
<span class="text-xs text-gray-500">{{.SourceID}}</span>
{{if gt .Count 1}}<span class="text-xs text-amber-500" title="seen {{.Count}} times">×{{.Count}}</span>{{end}}
<span class="text-xs text-gray-600">{{humanTime .CollectedAt}}</span>
</div>
{{end}}
</div>
</section>
{{end}}
{{if .notes}}
<section>
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Notes</h3>
<div class="space-y-2">
{{range .notes}}
<div class="bg-gray-900 rounded p-3 border border-gray-800">
<div class="text-xs text-gray-500">{{humanTime .CreatedAt}}</div>
<div class="text-sm mt-1">{{.Note}}</div>
</div>
{{end}}
</div>
</section>
{{end}}
</main>
{{template "foot" .}}
{{end}}
{{define "entry_detail"}}
{{template "head" .}}
<main class="max-w-4xl mx-auto p-6 space-y-4">
<div class="bg-gray-900 rounded-lg p-4 border border-gray-800">
<div class="grid grid-cols-2 gap-4 text-sm">
<div><span class="text-gray-500">Fingerprint</span><br><span class="font-mono text-xs">{{.entry.Fingerprint}}</span></div>
<div><span class="text-gray-500">Source</span><br>{{.entry.SourceID}}</div>
<div><span class="text-gray-500">Title</span><br><span class="font-medium">{{.entry.Title}}</span></div>
<div><span class="text-gray-500">Project</span><br>{{.entry.Project}}</div>
<div><span class="text-gray-500">Path</span><br><span class="font-mono text-xs">{{.entry.SourcePath}}</span></div>
<div><span class="text-gray-500">Type</span><br>{{.entry.ContentType}}</div>
<div><span class="text-gray-500">Signal Time</span><br>{{if .entry.CreatedAt}}{{humanTime .entry.CreatedAt}}{{else}}<span class="text-gray-500">—</span>{{end}}</div>
<div><span class="text-gray-500">Ingested</span><br>{{humanTime .entry.LastSeen}}</div>
<div><span class="text-gray-500">Ref Count</span><br>{{.entry.RefCount}}</div>
<div><span class="text-gray-500">Confidence</span><br>{{printf "%.1f" .entry.LastConfidence}}</div>
</div>
{{if .entry.Summary}}<div class="mt-4 p-3 bg-gray-800 rounded text-sm">{{.entry.Summary}}</div>{{end}}
</div>
{{if .obs}}
<section>
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Observation History ({{len .obs}})</h3>
<div class="space-y-1">
{{range .obs}}
<div class="bg-gray-900 rounded px-3 py-2 text-sm flex items-center gap-3">
<span class="font-mono text-xs text-gray-500">#{{.ID}}</span>
<span class="text-xs text-gray-500">{{humanTime .CollectedAt}}</span>
{{if gt .Count 1}}<span class="text-xs text-amber-500" title="seen {{.Count}} times, unchanged">×{{.Count}}</span>{{end}}
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.Trigger}}</span>
<span class="text-xs text-gray-500">{{.IngesterVersion}}</span>
<span class="text-xs text-gray-500">conf:{{printf "%.1f" .Confidence}}</span>
</div>
{{end}}
</div>
</section>
{{end}}
</main>
{{template "foot" .}}
{{end}}
+283
View File
@@ -0,0 +1,283 @@
package server
import (
"bytes"
_ "embed"
"fmt"
"html/template"
"log"
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/david/knox/internal/db"
)
type Server struct {
kdb *db.KnoxDB
mux *http.ServeMux
tmpl *template.Template
}
func New(kdb *db.KnoxDB) *Server {
s := &Server{kdb: kdb, mux: http.NewServeMux()}
s.tmpl = template.Must(template.New("knox").Funcs(template.FuncMap{
"shortFP": shortFP,
"trunc": truncStr,
"fpURL": fpURL,
"humanTime": humanTime,
"displayTime": displayTime,
}).Parse(layout))
s.mux.HandleFunc("GET /", s.dashboard)
s.mux.HandleFunc("GET /search", s.search)
s.mux.HandleFunc("GET /threads", s.threadList)
s.mux.HandleFunc("GET /threads/{id}", s.threadDetail)
s.mux.HandleFunc("GET /entries/{fp}", s.entryDetail)
s.mux.HandleFunc("GET /stats", s.stats)
return s
}
func (s *Server) Serve(addr string) error {
log.Printf("[knox] web UI at http://%s", addr)
return http.ListenAndServe(addr, s.mux)
}
// ─── Dashboard ───────────────────────────────────────────────
func (s *Server) dashboard(w http.ResponseWriter, r *http.Request) {
gitea, _ := s.kdb.RecentEntriesBySource("gitea", 3)
obsidian, _ := s.kdb.RecentEntriesBySource("obsidian", 3)
skills, _ := s.kdb.RecentEntriesBySource("skills-catalog", 2)
session, _ := s.kdb.RecentEntriesBySource("opencode-session", 3)
recent, _ := s.kdb.RecentEntries(8)
goldenID, err := s.kdb.GoldenThreadID()
if err != nil {
http.Error(w, "failed to load golden thread", http.StatusInternalServerError)
return
}
var golden *db.Thread
if goldenID > 0 {
golden, _ = s.kdb.GetThread(goldenID)
}
s.render(w, "dashboard", map[string]any{
"gitea": gitea,
"obsidian": obsidian,
"skills": skills,
"session": session,
"recent": recent,
"golden": golden,
})
}
// ─── Search ──────────────────────────────────────────────────
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query().Get("q")
if q == "" {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Write([]byte("<p class='text-gray-400 text-sm'>Enter a query above.</p>"))
return
}
results, err := s.kdb.Search(q, 30)
if err != nil {
http.Error(w, "search failed", http.StatusInternalServerError)
return
}
s.render(w, "search_results", map[string]any{
"query": q,
"results": results,
})
}
// ─── Thread List ─────────────────────────────────────────────
func (s *Server) threadList(w http.ResponseWriter, r *http.Request) {
status := r.URL.Query().Get("status")
threads, err := s.kdb.ListThreads(status)
if err != nil {
http.Error(w, "failed to load threads", http.StatusInternalServerError)
return
}
s.render(w, "thread_list", map[string]any{
"title": "Threads",
"threads": threads,
"status": status,
})
}
// ─── Thread Detail ───────────────────────────────────────────
func (s *Server) threadDetail(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err != nil {
http.Error(w, "invalid thread id", http.StatusBadRequest)
return
}
t, err := s.kdb.GetThread(id)
if err != nil {
http.Error(w, "failed to load thread", http.StatusInternalServerError)
return
}
if t == nil {
s.render(w, "not_found", map[string]any{"title": "Thread Not Found"}, http.StatusNotFound)
return
}
obs, err := s.kdb.ThreadObservations(id)
if err != nil {
http.Error(w, "failed to load observations", http.StatusInternalServerError)
return
}
notes, err := s.kdb.ThreadNotes(id)
if err != nil {
http.Error(w, "failed to load notes", http.StatusInternalServerError)
return
}
provenance, err := s.kdb.ThreadProvenance(id)
if err != nil {
http.Error(w, "failed to load provenance", http.StatusInternalServerError)
return
}
goldenID, _ := s.kdb.GoldenThreadID()
s.render(w, "thread_detail", map[string]any{
"title": fmt.Sprintf("Thread #%d", id),
"thread": t,
"obs": obs,
"notes": notes,
"provenance": provenance,
"isGolden": goldenID == id,
})
}
// ─── Entry Detail ────────────────────────────────────────────
func (s *Server) entryDetail(w http.ResponseWriter, r *http.Request) {
fp := r.PathValue("fp")
entry, err := s.kdb.FindEntry(fp)
if err != nil {
http.Error(w, "failed to load entry", http.StatusInternalServerError)
return
}
if entry == nil {
// Try fingerprint prefix match (the UI displays 8-char prefixes)
entries, err := s.kdb.FindEntryByPrefix(fp, 2)
if err != nil {
http.Error(w, "failed to load entry", http.StatusInternalServerError)
return
}
if len(entries) == 1 {
entry = &entries[0]
} else {
s.render(w, "not_found", map[string]any{"title": "Entry Not Found"}, http.StatusNotFound)
return
}
}
obs, err := s.kdb.ObservationsByFingerprint(entry.Fingerprint, 20)
if err != nil {
http.Error(w, "failed to load observations", http.StatusInternalServerError)
return
}
s.render(w, "entry_detail", map[string]any{
"title": "Entry " + shortFP(entry.Fingerprint),
"entry": entry,
"obs": obs,
})
}
// ─── Stats Fragment ──────────────────────────────────────────
type statItem struct {
Label string
Value any
}
func statItems(stats map[string]any) []statItem {
keys := make([]string, 0, len(stats))
for k := range stats {
keys = append(keys, k)
}
sort.Strings(keys)
items := make([]statItem, 0, len(keys))
for _, k := range keys {
items = append(items, statItem{
Label: strings.ReplaceAll(k, "_", " "),
Value: stats[k],
})
}
return items
}
func (s *Server) stats(w http.ResponseWriter, r *http.Request) {
stats, err := s.kdb.Stats()
if err != nil {
http.Error(w, "failed to load stats", http.StatusInternalServerError)
return
}
s.render(w, "stats", map[string]any{"stats": statItems(stats)})
}
// ─── Render ──────────────────────────────────────────────────
func (s *Server) render(w http.ResponseWriter, name string, data map[string]any, status ...int) {
code := http.StatusOK
if len(status) > 0 {
code = status[0]
}
var buf bytes.Buffer
if err := s.tmpl.ExecuteTemplate(&buf, name, data); err != nil {
log.Printf("[knox] template error: %v", err)
http.Error(w, "template error", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(code)
buf.WriteTo(w)
}
func shortFP(fp string) string {
if len(fp) > 8 {
return fp[:8]
}
return fp
}
// fpURL routes synthetic thread fingerprints to thread pages.
func fpURL(fp string) string {
if id, ok := strings.CutPrefix(fp, "thread:"); ok {
return "/threads/" + id
}
return "/entries/" + fp
}
func truncStr(s string, n int) string {
runes := []rune(s)
if len(runes) <= n {
return s
}
return string(runes[:n]) + "..."
}
func displayTime(e db.Entry) string {
if e.CreatedAt != "" {
return humanTime(e.CreatedAt)
}
return humanTime(e.LastSeen)
}
func humanTime(s string) string {
formats := []string{time.RFC3339, "2006-01-02 15:04:05"}
for _, f := range formats {
t, err := time.Parse(f, s)
if err == nil {
return t.Local().Format("Jan 2, 2006 · 15:04")
}
}
return s
}
//go:embed layout.html
var layout string
+311
View File
@@ -0,0 +1,311 @@
package watch
import (
"encoding/json"
"fmt"
"log"
"os"
"strconv"
"strings"
"time"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/index"
)
// AutoThreader promotes high-intent topic clusters into knowledge threads.
//
// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the
// "intent bar" when it shows sustained, recently-observed activity from a
// non-passive work source (git/gitea/session/obsidian) plus enough distinct
// sources overall. Motivation and title are synthesized from the cluster's
// strongest members — they can be refined later, but the grouping and linkage
// are deterministic and cheap enough for the watch daemon's periodic tick.
type AutoThreader struct {
DB *db.KnoxDB
MinSources int // distinct sources required (incl. one work source)
MinEntries int // minimum cluster size
Recency time.Duration // observations must be seen within this window
DryRun bool
}
// workSources are signal sources that imply active intent (vs passive browsing
// or an always-present skills catalog).
var workSources = map[string]bool{
"git": true,
"gitea": true,
"opencode-session": true,
"obsidian": true,
"filesystem": true,
}
// NewAutoThreader reads the tuning knobs from the environment. All have
// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off.
func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader {
t := &AutoThreader{
DB: kdb,
MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2),
MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3),
Recency: time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour,
}
return t
}
func envInt(key string, def int) int {
if v, err := strconv.Atoi(os.Getenv(key)); err == nil {
return v
}
return def
}
// AutoThread clusters recent entries and creates/extends threads for clusters
// crossing the intent bar. Returns the number of threads created and the total
// observations linked.
func (t *AutoThreader) AutoThread() (created, linked int, err error) {
entries, err := t.DB.RecentEntries(2000)
if err != nil {
return 0, 0, err
}
if len(entries) == 0 {
return 0, 0, nil
}
tfidf := index.BuildTFIDF(entries)
clusters := tfidf.Cluster(2, 30)
now := time.Now().UTC()
for _, c := range clusters {
fps := fingerprints(c.Entries)
if len(fps) == 0 || !t.crossesBar(c, fps, now) {
continue
}
// Fold into an existing active thread if one already covers these terms.
if existing := t.findExisting(c); existing > 0 {
if t.DryRun {
fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name)
continue
}
n, err := t.DB.AutoLinkThreadObservations(existing, fps)
if err != nil {
log.Printf("[knox] threader link err: %v", err)
continue
}
if n > 0 {
log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n)
}
linked += n
continue
}
title := t.title(c)
motivation := t.motivation(c, fps)
tags := t.tags(c)
priority := t.priority(c)
prov := map[string]any{
"trigger": "auto_thread",
"cluster": c.Name,
"score": c.Score,
"sources": sourceList(c.Entries),
}
provJSON, _ := json.Marshal(prov)
if t.DryRun {
fmt.Printf("[threader] (dry-run) would create thread %q [%s]\n", title, priority)
fmt.Printf(" motivation: %s\n", motivation)
fmt.Printf(" tags: %s\n", tags)
fmt.Printf(" %d obs | sources: %v\n", len(fps), prov["sources"])
continue
}
id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON))
if err != nil {
log.Printf("[knox] threader create err: %v", err)
continue
}
if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil {
linked += n
}
created++
log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority)
}
return created, linked, nil
}
// crossesBar decides whether a cluster reflects real intent.
func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool {
if len(c.Entries) < t.MinEntries {
return false
}
bySource := make(map[string]bool)
hasWork := false
recent := false
for _, e := range c.Entries {
if e.SourceID != "" {
bySource[e.SourceID] = true
if workSources[e.SourceID] {
hasWork = true
}
}
if e.LastSeen != "" {
if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency {
recent = true
}
}
}
if len(bySource) < t.MinSources {
return false
}
if !hasWork {
return false
}
return recent
}
// findExisting looks for an already-active thread that plausibly covers this
// cluster, by matching a discriminative cluster keyword against thread titles.
func (t *AutoThreader) findExisting(c index.TopicCluster) int64 {
for _, kw := range c.Keywords {
id, err := t.DB.ActiveThreadByKeyword(kw)
if err != nil {
return 0
}
if id == 0 {
continue
}
th, err := t.DB.GetThread(id)
if err != nil || th == nil {
continue
}
titleTokens := toSet(strings.ToLower(th.Title))
for _, kw2 := range c.Keywords {
if titleTokens[kw2] {
return id
}
}
}
return 0
}
func (t *AutoThreader) title(c index.TopicCluster) string {
return c.Name
}
func (t *AutoThreader) tags(c index.TopicCluster) string {
var keep []string
for _, kw := range c.Keywords {
if len(keep) >= 5 {
break
}
keep = append(keep, kw)
}
return strings.Join(keep, " ")
}
func (t *AutoThreader) priority(c index.TopicCluster) string {
srcs := len(sourceList(c.Entries))
if srcs >= 3 {
return "high"
}
return "medium"
}
// motivation synthesizes "why" from the cluster's strongest members: project
// context, source spread, and a couple of representative titles/summaries.
func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string {
sources := sourceList(c.Entries)
proj := dominantProject(c.Entries)
var b strings.Builder
fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", "))
if proj != "" {
fmt.Fprintf(&b, " Dominant project: %s.", proj)
}
fmt.Fprintf(&b, " Keywords: %s.", c.Name)
fmt.Fprintf(&b, " Representative observations:")
shown := 0
seen := make(map[string]bool)
for _, e := range c.Entries {
if shown >= 3 {
break
}
txt := strings.TrimSpace(e.Title)
if txt == "" {
txt = strings.TrimSpace(e.Summary)
}
if txt == "" || seen[txt] {
continue
}
seen[txt] = true
fmt.Fprintf(&b, " • %s", truncate(txt, 90))
shown++
}
return b.String()
}
// --- small helpers ---------------------------------------------------------
func fingerprints(entries []db.Entry) []string {
seen := make(map[string]bool)
var out []string
for _, e := range entries {
if e.Fingerprint == "" || seen[e.Fingerprint] {
continue
}
seen[e.Fingerprint] = true
out = append(out, e.Fingerprint)
}
return out
}
func sourceList(entries []db.Entry) []string {
seen := make(map[string]bool)
var out []string
for _, e := range entries {
if e.SourceID == "" || seen[e.SourceID] {
continue
}
seen[e.SourceID] = true
out = append(out, e.SourceID)
}
return out
}
func dominantProject(entries []db.Entry) string {
counts := make(map[string]int)
for _, e := range entries {
if e.Project != "" {
counts[e.Project]++
}
}
best, n := "", 0
for p, c := range counts {
if c > n {
best, n = p, c
}
}
if n*2 >= len(entries) {
return best
}
return ""
}
func toSet(s string) map[string]bool {
set := make(map[string]bool)
for _, w := range strings.Fields(s) {
set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true
}
return set
}
func truncate(s string, n int) string {
r := []rune(s)
if len(r) <= n {
return s
}
return string(r[:n]) + "…"
}
+447
View File
@@ -0,0 +1,447 @@
package watch
import (
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/david/knox/internal/db"
"github.com/david/knox/internal/ingest"
"github.com/fsnotify/fsnotify"
)
const browserInterval = 5 * time.Minute
type Watcher struct {
knoxDB *db.KnoxDB
dirs []string
vault string
debounce time.Duration
fileIngesters []ingest.Ingester
}
func New(kdb *db.KnoxDB, dirs []string) *Watcher {
// Detect Obsidian vault
vault, _ := ingest.DetectObsidianVault()
return &Watcher{
knoxDB: kdb,
dirs: dirs,
vault: vault,
debounce: 2 * time.Second,
fileIngesters: []ingest.Ingester{
ingest.NewSessionDiffIngester(),
ingest.NewLogIngester(),
ingest.NewSkillsIngester(),
},
}
}
func (w *Watcher) Start() error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return err
}
defer watcher.Close()
w.seed(watcher)
log.Printf("[knox] watching %d directories", len(w.dirs))
if w.vault != "" {
log.Printf("[knox] obsidian vault: %s", w.vault)
}
debounceMap := make(map[string]time.Time)
browserTicker := time.NewTicker(browserInterval)
giteaTicker := time.NewTicker(10 * time.Minute)
gitTicker := time.NewTicker(10 * time.Minute)
threadTicker := time.NewTicker(10 * time.Minute)
browserRunning := false
giteaRunning := false
gitRunning := false
threadRunning := false
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return nil
}
if !w.isRelevantEvent(event) {
continue
}
now := time.Now()
if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce {
continue
}
debounceMap[event.Name] = now
trigger := eventOpName(event.Op)
log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
w.ingestFile(event.Name, trigger)
case <-browserTicker.C:
if browserRunning {
continue
}
browserRunning = true
go func() {
defer func() { browserRunning = false }()
w.ingestBrowserHistory()
}()
case <-giteaTicker.C:
if giteaRunning {
continue
}
giteaRunning = true
go func() {
defer func() { giteaRunning = false }()
w.ingestGitea()
}()
case <-gitTicker.C:
if gitRunning {
continue
}
gitRunning = true
go func() {
defer func() { gitRunning = false }()
w.ingestGit()
}()
case <-threadTicker.C:
if threadRunning {
continue
}
threadRunning = true
go func() {
defer func() { threadRunning = false }()
w.autoThread()
}()
case err, ok := <-watcher.Errors:
if !ok {
return nil
}
log.Printf("[knox] watch error: %v", err)
}
}
}
func (w *Watcher) seed(watcher *fsnotify.Watcher) {
for _, dir := range w.dirs {
abs, _ := filepath.Abs(dir)
if err := watcher.Add(abs); err != nil {
log.Printf("[knox] cannot watch %s: %v", abs, err)
continue
}
log.Printf("[knox] watching %s", abs)
}
// Watch Obsidian vault
if w.vault != "" {
if err := watcher.Add(w.vault); err != nil {
log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
} else {
log.Printf("[knox] watching %s (obsidian)", w.vault)
}
}
// Seed existing files
for _, ing := range w.fileIngesters {
for _, dir := range w.dirs {
patterns := []string{
filepath.Join(dir, "*"),
filepath.Join(dir, "*", "SKILL.md"),
}
for _, pattern := range patterns {
entries, _ := filepath.Glob(pattern)
for _, path := range entries {
if !MatchesIngester(path, ing.SourceID()) {
continue
}
w.ingestFileWith(path, ing, "seed")
}
}
}
}
// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
// and covers all depths — glob patterns would match dot-dirs and miss depth >2.
if w.vault != "" {
if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
for _, r := range notes {
w.recordResult(r, "seed")
}
}
}
}
func (w *Watcher) ingestFile(path, trigger string) {
for _, ing := range w.fileIngesters {
if MatchesIngester(path, ing.SourceID()) {
w.ingestFileWith(path, ing, trigger)
return
}
}
// Obsidian: any .md file in the vault
if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
obs := ingest.NewObsidianFileIngester(w.vault)
w.ingestFileWith(path, obs, trigger)
}
}
func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
result, err := ing.Ingest(path)
if err != nil {
log.Printf("[knox] ingest error %s: %v", path, err)
return
}
if result == nil {
return
}
// Staleness check: skip when the signal time is known and unchanged.
// Empty CreatedAt must NOT skip — it would suppress everything.
if result.CreatedAt != "" {
if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
return
}
}
w.recordResult(result, trigger)
}
// recordResult persists one ingest result and fires side effects (session
// tracking, golden-thread linking happens inside RecordObservation).
func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
LineStart: result.LineStart,
LineEnd: result.LineEnd,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: trigger,
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
log.Printf("[knox] db error: %v", err)
return
}
if isNew {
log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
} else {
log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
}
if result.SourceID == "opencode-session" {
sessionID, _ := result.Provenance["session_id"].(string)
if sessionID != "" {
status := "active"
w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
}
}
}
func (w *Watcher) ingestBrowserHistory() {
log.Printf("[knox] periodic browser history ingest...")
ing := ingest.NewBrowserHistoryIngester()
results, err := ing.IngestAll()
if err != nil {
log.Printf("[knox] browser ingest error: %v", err)
return
}
var newCount, skipped int
for _, result := range results {
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
if existing != nil && existing.CreatedAt == result.CreatedAt {
skipped++
continue
}
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "browser_timer",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
continue
}
if isNew {
newCount++
}
}
if newCount > 0 || skipped > 0 {
log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
}
}
func (w *Watcher) ingestGitea() {
log.Printf("[knox] periodic gitea ingest...")
ing := ingest.NewGiteaIngester()
results, err := ing.IngestAll()
if err != nil {
log.Printf("[knox] gitea ingest error: %v", err)
return
}
var newCount, skipped int
for _, result := range results {
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
if existing != nil && existing.CreatedAt == result.CreatedAt {
skipped++
continue
}
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "gitea_timer",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
continue
}
if isNew {
newCount++
}
}
if newCount > 0 || skipped > 0 {
log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
}
}
func (w *Watcher) ingestGit() {
log.Printf("[knox] periodic git status ingest...")
ing := ingest.NewGitIngester()
ing.Recursive = true
if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
for _, part := range strings.Split(v, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
ing.Roots = append(ing.Roots, part)
}
}
results, err := ing.IngestAll()
if err != nil {
log.Printf("[knox] git ingest error: %v", err)
return
}
var newCount, skipped int
for _, result := range results {
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
if existing != nil && existing.CreatedAt == result.CreatedAt {
skipped++
continue
}
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
Fingerprint: result.Fingerprint,
SourceID: result.SourceID,
SourcePath: result.SourcePath,
Project: result.Project,
ContentType: result.ContentType,
Title: result.Title,
Summary: result.Summary,
CreatedAt: result.CreatedAt,
Confidence: result.Confidence,
IngesterVersion: result.IngesterVersion,
Trigger: "git_timer",
Provenance: ingest.ProvenanceJSON(result.Provenance),
})
if err != nil {
continue
}
if isNew {
newCount++
}
}
if newCount > 0 || skipped > 0 {
log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
}
}
// autoThread runs the heuristic auto-threader. Disabled entirely when
// KNOX_THREAD_DISABLE is non-empty.
func (w *Watcher) autoThread() {
if os.Getenv("KNOX_THREAD_DISABLE") != "" {
return
}
threader := NewAutoThreader(w.knoxDB)
created, linked, err := threader.AutoThread()
if err != nil {
log.Printf("[knox] auto-thread error: %v", err)
return
}
if created > 0 || linked > 0 {
log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
}
}
func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
return false
}
name := filepath.Base(event.Name)
// Opencode session diffs and logs
if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
return true
}
// Obsidian markdown files
if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
return true
}
return false
}
func MatchesIngester(path, sourceID string) bool {
base := filepath.Base(path)
switch sourceID {
case "opencode-session":
return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
case "opencode-log":
return strings.HasSuffix(base, ".log")
case "skills-catalog":
return base == "SKILL.md"
}
return false
}
func eventOpName(op fsnotify.Op) string {
switch {
case op.Has(fsnotify.Create):
return "inotify:CREATE"
case op.Has(fsnotify.Write):
return "inotify:WRITE"
case op.Has(fsnotify.Chmod):
return "inotify:CHMOD"
default:
return "inotify:UNKNOWN"
}
}