Initial commit: knox knowledge index
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
)
|
||||
|
||||
func NewBrowserCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "browser",
|
||||
Short: "Ingest browser history from Chromium and Min into the knowledge index",
|
||||
Long: `Reads browser history databases (Chromium and Min), fingerprints URLs,
|
||||
and stores them as observations. Tracks what you've searched for and read.
|
||||
|
||||
Run periodically to capture browsing exhaust.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
ing := ingest.NewBrowserHistoryIngester()
|
||||
log.Printf("[knox] ingesting browser history...")
|
||||
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("browser ingest: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
fmt.Println("No browser history found (Chromium/Min not detected or empty).")
|
||||
return nil
|
||||
}
|
||||
|
||||
var newCount, dupCount int
|
||||
for _, result := range results {
|
||||
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "browser_ingest",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
} else {
|
||||
dupCount++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Browser history: %d new, %d already seen\n", newCount, dupCount)
|
||||
|
||||
// Show top domains
|
||||
domains := make(map[string]int)
|
||||
for _, r := range results {
|
||||
domains[extractDomain(r.Summary)]++
|
||||
}
|
||||
fmt.Println("\nTop domains:")
|
||||
for _, d := range topDomains(domains, 10) {
|
||||
fmt.Printf(" %-40s %d\n", d.Name, d.Count)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
func topDomains(domains map[string]int, n int) []struct {
|
||||
Name string
|
||||
Count int
|
||||
} {
|
||||
var sorted []struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
for name, count := range domains {
|
||||
sorted = append(sorted, struct {
|
||||
Name string
|
||||
Count int
|
||||
}{name, count})
|
||||
}
|
||||
// Simple bubble sort for small n
|
||||
for i := 0; i < len(sorted)-1; i++ {
|
||||
for j := 0; j < len(sorted)-1-i; j++ {
|
||||
if sorted[j].Count < sorted[j+1].Count {
|
||||
sorted[j], sorted[j+1] = sorted[j+1], sorted[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(sorted) > n {
|
||||
sorted = sorted[:n]
|
||||
}
|
||||
return sorted
|
||||
}
|
||||
|
||||
func extractDomain(url string) string {
|
||||
url = strings.TrimPrefix(url, "https://")
|
||||
url = strings.TrimPrefix(url, "http://")
|
||||
parts := strings.Split(url, "/")
|
||||
if len(parts) > 0 {
|
||||
return parts[0]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func NewGitCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var roots []string
|
||||
var recursive bool
|
||||
cmd := &cobra.Command{
|
||||
Args: cobra.NoArgs,
|
||||
Use: "git",
|
||||
Short: "Ingest local git repository status into the knowledge index",
|
||||
Long: `Scans git repos under one or more root dirs (default ~/src) and records
|
||||
each project's status — branch, clean/dirty, ahead/behind, HEAD, last commit.
|
||||
With --recursive (default) repos are found at any depth, following symlinked
|
||||
directories, so nested repos like ~/assistant/kitmaker are picked up. Use
|
||||
'knox git' for a one-shot ingest, or rely on the watch daemon's git timer.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
ing := &ingest.GitIngester{Roots: roots, Recursive: recursive}
|
||||
log.Printf("[knox] ingesting git repo status...")
|
||||
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("git ingest: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
fmt.Println("No git repos found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
ingest.SortResults(results)
|
||||
var newCount int
|
||||
for _, result := range results {
|
||||
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "git_ingest",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
}
|
||||
fmt.Printf("%-24s %s\n", result.Project, result.Summary)
|
||||
}
|
||||
fmt.Printf("\nGit: %d repos (%d new)\n", len(results), newCount)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringSliceVarP(&roots, "root", "r", nil, "Directories to scan for git repos (default ~/src; repeat or comma-separate for multiple roots)")
|
||||
cmd.Flags().BoolVar(&recursive, "recursive", true, "Recurse into subdirectories (following symlinks) to find nested git repos")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
)
|
||||
|
||||
func NewGiteaCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "gitea",
|
||||
Short: "Ingest Gitea repositories, issues, and PRs into the knowledge index",
|
||||
Long: `Uses the 'tea' CLI to fetch repositories, open issues, and open
|
||||
pull requests from Gitea. Fingerprints each by ID for dedup.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
ing := ingest.NewGiteaIngester()
|
||||
log.Printf("[knox] ingesting Gitea data via tea...")
|
||||
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("gitea ingest: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
fmt.Println("No Gitea data found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
var repos, issues, pulls int
|
||||
var newCount int
|
||||
for _, result := range results {
|
||||
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "gitea_ingest",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
}
|
||||
switch result.ContentType {
|
||||
case "repo":
|
||||
repos++
|
||||
case "issue":
|
||||
issues++
|
||||
case "pull":
|
||||
pulls++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Gitea: %d repos, %d issues, %d PRs (%d new)\n", repos, issues, pulls, newCount)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
"github.com/david/knox/internal/watch"
|
||||
)
|
||||
|
||||
func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var dirs []string
|
||||
cmd := &cobra.Command{
|
||||
Use: "ingest",
|
||||
Short: "One-time full sweep of all sources into the knowledge index",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(dirs) == 0 {
|
||||
dirs = defaultWatchDirs()
|
||||
}
|
||||
|
||||
ingesters := []ingest.Ingester{
|
||||
ingest.NewSessionDiffIngester(),
|
||||
ingest.NewLogIngester(),
|
||||
ingest.NewSkillsIngester(),
|
||||
}
|
||||
|
||||
var totalNew, totalUpdated int
|
||||
|
||||
record := func(result *ingest.IngestResult) {
|
||||
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
LineStart: result.LineStart,
|
||||
LineEnd: result.LineEnd,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "ingest",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
return
|
||||
}
|
||||
if isNew {
|
||||
totalNew++
|
||||
} else {
|
||||
totalUpdated++
|
||||
}
|
||||
|
||||
if result.SourceID == "opencode-session" {
|
||||
sessionID, _ := result.Provenance["session_id"].(string)
|
||||
if sessionID != "" {
|
||||
kdb.UpsertSession(sessionID, result.Project, result.Title, "active")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, ing := range ingesters {
|
||||
sid := ing.SourceID()
|
||||
log.Printf("[knox] ingesting %s...", sid)
|
||||
|
||||
for _, dir := range dirs {
|
||||
patterns := []string{
|
||||
dir + "/*",
|
||||
dir + "/*/SKILL.md",
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
entries, _ := filepath.Glob(pattern)
|
||||
for _, path := range entries {
|
||||
if !watch.MatchesIngester(path, sid) {
|
||||
continue
|
||||
}
|
||||
|
||||
result, err := ing.Ingest(path)
|
||||
if err != nil || result == nil {
|
||||
continue
|
||||
}
|
||||
record(result)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Non-file sources: obsidian vault, browser history, gitea
|
||||
if vault, err := ingest.DetectObsidianVault(); err == nil {
|
||||
log.Printf("[knox] ingesting obsidian...")
|
||||
if results, err := ingest.NewObsidianIngester(vault).IngestAll(); err == nil {
|
||||
for _, r := range results {
|
||||
record(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Printf("[knox] ingesting browser-history...")
|
||||
if results, err := ingest.NewBrowserHistoryIngester().IngestAll(); err == nil {
|
||||
for _, r := range results {
|
||||
record(r)
|
||||
}
|
||||
}
|
||||
log.Printf("[knox] ingesting gitea...")
|
||||
if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
|
||||
for _, r := range results {
|
||||
record(r)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to scan (default: opencode storage/log + skills)")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,655 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/index"
|
||||
"github.com/mark3labs/mcp-go/mcp"
|
||||
"github.com/mark3labs/mcp-go/server"
|
||||
)
|
||||
|
||||
type detailLevel int
|
||||
|
||||
const (
|
||||
detailBrief detailLevel = iota
|
||||
detailNormal
|
||||
detailFull
|
||||
)
|
||||
|
||||
func parseDetail(s string) detailLevel {
|
||||
switch strings.ToLower(s) {
|
||||
case "brief":
|
||||
return detailBrief
|
||||
case "full":
|
||||
return detailFull
|
||||
default:
|
||||
return detailNormal
|
||||
}
|
||||
}
|
||||
|
||||
func NewMCPServer(kdb *db.KnoxDB) *server.MCPServer {
|
||||
s := server.NewMCPServer(
|
||||
"knox-knowledge",
|
||||
"0.2.0",
|
||||
server.WithResourceCapabilities(true, true),
|
||||
server.WithPromptCapabilities(true),
|
||||
server.WithLogging(),
|
||||
)
|
||||
|
||||
// ─── knox_search ──────────────────────────────────────────
|
||||
searchTool := mcp.NewTool("knox_search",
|
||||
mcp.WithDescription("Search the knowledge index. Supports zoom levels: brief (lean), normal (default), full (verbose). Scope filters: all, skills, sessions, logs."),
|
||||
mcp.WithString("query", mcp.Required(), mcp.Description("Search query")),
|
||||
mcp.WithNumber("limit", mcp.Description("Max results (default 10)")),
|
||||
mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")),
|
||||
mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")),
|
||||
)
|
||||
|
||||
s.AddTool(searchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
query, _ := req.Params.Arguments["query"].(string)
|
||||
limit := clampInt(req, "limit", 10, 1, 50)
|
||||
detail := parseDetail(getString(req, "detail", "normal"))
|
||||
scope := getString(req, "scope", "all")
|
||||
|
||||
results, err := kdb.Search(query, limit)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
results = filterByScope(results, scope)
|
||||
if len(results) == 0 {
|
||||
return mcp.NewToolResultText(fmt.Sprintf("No results for %q in scope %q", query, scope)), nil
|
||||
}
|
||||
|
||||
return mcp.NewToolResultText(formatEntries(results, detail, kdb)), nil
|
||||
})
|
||||
|
||||
// ─── knox_recent ───────────────────────────────────────────
|
||||
recentTool := mcp.NewTool("knox_recent",
|
||||
mcp.WithDescription("Show recent knowledge entries with zoom levels: brief, normal (default), full."),
|
||||
mcp.WithNumber("limit", mcp.Description("Max results (default 10)")),
|
||||
mcp.WithString("detail", mcp.Description("Zoom level: brief, normal, or full (default normal)")),
|
||||
mcp.WithString("scope", mcp.Description("Source scope: all, skills, sessions, logs (default all)")),
|
||||
)
|
||||
|
||||
s.AddTool(recentTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
limit := clampInt(req, "limit", 10, 1, 50)
|
||||
detail := parseDetail(getString(req, "detail", "normal"))
|
||||
scope := getString(req, "scope", "all")
|
||||
|
||||
entries, err := kdb.RecentEntries(limit)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
entries = filterByScope(entries, scope)
|
||||
if len(entries) == 0 {
|
||||
return mcp.NewToolResultText("No entries yet."), nil
|
||||
}
|
||||
|
||||
return mcp.NewToolResultText(formatEntries(entries, detail, kdb)), nil
|
||||
})
|
||||
|
||||
// ─── knox_get ──────────────────────────────────────────────
|
||||
getTool := mcp.NewTool("knox_get",
|
||||
mcp.WithDescription("Get full detail for a specific entry by fingerprint (drill-down from search results)."),
|
||||
mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint (full or first 8 chars)")),
|
||||
)
|
||||
|
||||
s.AddTool(getTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
fp := getString(req, "fingerprint", "")
|
||||
|
||||
entry, err := kdb.FindEntry(fp)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if entry == nil {
|
||||
// Try fingerprint prefix match
|
||||
entries, err := kdb.FindEntryByPrefix(fp, 5)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if len(entries) == 1 {
|
||||
entry = &entries[0]
|
||||
} else if len(entries) > 1 {
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Multiple matches for %q — use full fingerprint:\n%s", fp, formatEntriesBrief(entries))), nil
|
||||
} else {
|
||||
return mcp.NewToolResultText(fmt.Sprintf("No entry found for fingerprint %q", fp)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Full detail: entry + observations + provenance chain
|
||||
var b strings.Builder
|
||||
b.WriteString(formatEntryFull(entry))
|
||||
|
||||
obs, _ := kdb.ObservationsByFingerprint(entry.Fingerprint, 20)
|
||||
if len(obs) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\nObservations (%d):\n", len(obs)))
|
||||
for _, o := range obs {
|
||||
b.WriteString(fmt.Sprintf(" [%s] trigger=%s confidence=%.1f line=%d ver=%s\n",
|
||||
o.CollectedAt, o.Trigger, o.Confidence, o.LineEnd, o.IngesterVersion))
|
||||
}
|
||||
}
|
||||
|
||||
chain, _ := kdb.ProvenanceChain(entry.Fingerprint, 3)
|
||||
if len(chain) > 0 {
|
||||
b.WriteString(fmt.Sprintf("\nProvenance links (%d):\n", len(chain)))
|
||||
for _, l := range chain {
|
||||
b.WriteString(fmt.Sprintf(" %s --[%s]--> %s (hop %d)\n", shortFP(l.Fingerprint), l.Relation, shortFP(entry.Fingerprint), l.HopDistance))
|
||||
}
|
||||
}
|
||||
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_stats ────────────────────────────────────────────
|
||||
statsTool := mcp.NewTool("knox_stats",
|
||||
mcp.WithDescription("Get knowledge index statistics"),
|
||||
)
|
||||
|
||||
s.AddTool(statsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
stats, err := kdb.Stats()
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
b, _ := json.MarshalIndent(stats, "", " ")
|
||||
return mcp.NewToolResultText(string(b)), nil
|
||||
})
|
||||
|
||||
// ─── knox_count ────────────────────────────────────────────
|
||||
countTool := mcp.NewTool("knox_count",
|
||||
mcp.WithDescription("Return hit count matching a query, grouped by source. Lightweight — no result bodies, minimal tokens."),
|
||||
mcp.WithString("query", mcp.Required(), mcp.Description("Search query")),
|
||||
)
|
||||
|
||||
s.AddTool(countTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
query, _ := req.Params.Arguments["query"].(string)
|
||||
results, err := kdb.Search(query, 500)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
total := len(results)
|
||||
bySource := make(map[string]int)
|
||||
for _, r := range results {
|
||||
bySource[r.SourceID]++
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("Matches for %q: %d total\n", query, total))
|
||||
for src, count := range bySource {
|
||||
b.WriteString(fmt.Sprintf(" %-20s %d\n", src+":", count))
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_reflect ──────────────────────────────────────────
|
||||
reflectTool := mcp.NewTool("knox_reflect",
|
||||
mcp.WithDescription("Process pending sessions for reflection and gap analysis"),
|
||||
)
|
||||
|
||||
s.AddTool(reflectTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
reflector := index.NewReflector(kdb)
|
||||
results, err := reflector.Reflect()
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return mcp.NewToolResultText("No pending sessions to reflect on."), nil
|
||||
}
|
||||
return mcp.NewToolResultText(reflector.FormatResult(results)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_list ───────────────────────────────────────
|
||||
threadListTool := mcp.NewTool("knox_thread_list",
|
||||
mcp.WithDescription("List knowledge development threads with their motivation, status, and linked entry count."),
|
||||
mcp.WithString("status", mcp.Description("Filter: active, stalled, resolved (default all)")),
|
||||
)
|
||||
|
||||
s.AddTool(threadListTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
status := getString(req, "status", "")
|
||||
threads, err := kdb.ListThreads(status)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if len(threads) == 0 {
|
||||
return mcp.NewToolResultText("No threads."), nil
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, t := range threads {
|
||||
b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount))
|
||||
if t.Motivation != "" {
|
||||
b.WriteString(fmt.Sprintf(" %s\n", t.Motivation))
|
||||
}
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_create ─────────────────────────────────────
|
||||
threadCreateTool := mcp.NewTool("knox_thread_create",
|
||||
mcp.WithDescription("Create a new knowledge development thread to track intent/motivation behind observations."),
|
||||
mcp.WithString("title", mcp.Required(), mcp.Description("Thread title")),
|
||||
mcp.WithString("motivation", mcp.Description("Why this thread exists (the 'so what')")),
|
||||
mcp.WithString("priority", mcp.Description("high, medium, or low (default medium)")),
|
||||
mcp.WithString("provenance", mcp.Description("JSON provenance metadata: {\"trigger\":\"...\",\"session_id\":\"...\"}")),
|
||||
)
|
||||
|
||||
s.AddTool(threadCreateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
title := getString(req, "title", "")
|
||||
motivation := getString(req, "motivation", "")
|
||||
priority := getString(req, "priority", "medium")
|
||||
provenance := getString(req, "provenance", "{}")
|
||||
id, err := kdb.CreateThread(title, motivation, priority, "[]", provenance)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Created thread #%d: %s", id, title)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_update ─────────────────────────────────────
|
||||
threadUpdateTool := mcp.NewTool("knox_thread_update",
|
||||
mcp.WithDescription("Rewrite a thread's editorial fields (title/motivation/priority/tags). Used to polish auto-generated threads."),
|
||||
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID to update")),
|
||||
mcp.WithString("title", mcp.Description("New title (leave empty to keep)")),
|
||||
mcp.WithString("motivation", mcp.Description("New motivation (leave empty to keep)")),
|
||||
mcp.WithString("priority", mcp.Description("New priority: high/medium/low (empty to keep)")),
|
||||
mcp.WithString("tags", mcp.Description("New space-separated tags (empty to keep)")),
|
||||
)
|
||||
|
||||
s.AddTool(threadUpdateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
id := int64(clampInt(req, "thread_id", 0, 1, 999999))
|
||||
title := getString(req, "title", "")
|
||||
motivation := getString(req, "motivation", "")
|
||||
priority := getString(req, "priority", "")
|
||||
tags := getString(req, "tags", "")
|
||||
changed, err := kdb.UpdateThread(id, title, motivation, priority, tags)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if !changed {
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Thread #%d unchanged (no edits differed).", id)), nil
|
||||
}
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Updated thread #%d", id)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_draft ──────────────────────────────────────
|
||||
// Emits auto-threaded candidates with enough context (sources, linked
|
||||
// observations) for an agent to write tight titles/motivations.
|
||||
threadDraftTool := mcp.NewTool("knox_thread_draft",
|
||||
mcp.WithDescription("List auto-threaded candidates needing LLM polish. Returns each heuristic thread's current title/motivation and its linked observations so you can rewrite them via knox_thread_update."),
|
||||
mcp.WithNumber("limit", mcp.Description("Max candidates to return (default 20)")),
|
||||
)
|
||||
|
||||
s.AddTool(threadDraftTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
limit := clampInt(req, "limit", 20, 1, 200)
|
||||
threads, err := kdb.ListThreads("")
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
selected := make([]db.Thread, 0, len(threads))
|
||||
for _, t := range threads {
|
||||
if strings.Contains(t.Provenance, "auto_thread") || strings.HasPrefix(t.Motivation, "Auto-detected") {
|
||||
selected = append(selected, t)
|
||||
}
|
||||
if len(selected) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(selected) == 0 {
|
||||
return mcp.NewToolResultText("No auto-threaded candidates. Run `knox thread auto` or wait for the watch tick first."), nil
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
for _, t := range selected {
|
||||
obs, _ := kdb.ThreadObservations(t.ID)
|
||||
fmt.Fprintf(&b, "THREAD #%d [%s] %d obs\n title: %s\n motivation: %s\n",
|
||||
t.ID, t.Priority, t.EntryCount, t.Title, t.Motivation)
|
||||
if t.Tags != "[]" {
|
||||
fmt.Fprintf(&b, " tags: %s\n", t.Tags)
|
||||
}
|
||||
b.WriteString(" linked observations:\n")
|
||||
for _, o := range obs {
|
||||
line := strings.TrimSpace(o.Title)
|
||||
if o.Summary != "" {
|
||||
line += " — " + strings.TrimSpace(o.Summary)
|
||||
}
|
||||
fmt.Fprintf(&b, " • [%s] %s\n", o.SourceID, truncateStr(line, 160))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_link ───────────────────────────────────────
|
||||
threadLinkTool := mcp.NewTool("knox_thread_link",
|
||||
mcp.WithDescription("Link an observation to a thread by observation ID. Use knox_get first to find observation IDs."),
|
||||
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")),
|
||||
mcp.WithNumber("observation_id", mcp.Required(), mcp.Description("Observation ID to link")),
|
||||
mcp.WithString("relevance", mcp.Description("Why this observation matters to the thread")),
|
||||
)
|
||||
|
||||
s.AddTool(threadLinkTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
threadID := int64(clampInt(req, "thread_id", 0, 1, 999999))
|
||||
obsID := int64(clampInt(req, "observation_id", 0, 1, 999999))
|
||||
relevance := getString(req, "relevance", "")
|
||||
if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Linked observation #%d to thread #%d", obsID, threadID)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_search ─────────────────────────────────────
|
||||
threadSearchTool := mcp.NewTool("knox_thread_search",
|
||||
mcp.WithDescription("Search threads by motivation, title, or tags. Find in-flight knowledge work by intent."),
|
||||
mcp.WithString("query", mcp.Required(), mcp.Description("Search motivation, title, or tags")),
|
||||
)
|
||||
|
||||
s.AddTool(threadSearchTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
query := getString(req, "query", "")
|
||||
threads, err := kdb.SearchThreadsByMotivation(query)
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
if len(threads) == 0 {
|
||||
return mcp.NewToolResultText(fmt.Sprintf("No threads matching %q", query)), nil
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, t := range threads {
|
||||
b.WriteString(fmt.Sprintf("#%d %-40s [%s/%s] %d entries\n", t.ID, t.Title, t.Status, t.Priority, t.EntryCount))
|
||||
if t.Motivation != "" {
|
||||
b.WriteString(fmt.Sprintf(" %s\n", t.Motivation))
|
||||
}
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_link_entry ─────────────────────────────────
|
||||
threadLinkEntryTool := mcp.NewTool("knox_thread_link_entry",
|
||||
mcp.WithDescription("Link an entry to a thread via provenance graph. Use after knox_search to connect findings to intent."),
|
||||
mcp.WithNumber("thread_id", mcp.Required(), mcp.Description("Thread ID")),
|
||||
mcp.WithString("fingerprint", mcp.Required(), mcp.Description("Entry fingerprint")),
|
||||
mcp.WithString("relation", mcp.Description("Relation type: produced (default), references, motivated")),
|
||||
)
|
||||
|
||||
s.AddTool(threadLinkEntryTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
threadID := int64(clampInt(req, "thread_id", 0, 1, 999999))
|
||||
fp := getString(req, "fingerprint", "")
|
||||
relation := getString(req, "relation", "produced")
|
||||
if err := kdb.LinkEntryToThread(threadID, fp, relation); err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Linked %s to thread #%d [%s]", shortFP(fp), threadID, relation)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_relate ─────────────────────────────────────
|
||||
threadRelateTool := mcp.NewTool("knox_thread_relate",
|
||||
mcp.WithDescription("Relate two threads (parent spawns child, threads merge, etc.). Builds the thread-inquiry graph."),
|
||||
mcp.WithNumber("parent_id", mcp.Required(), mcp.Description("Parent thread ID")),
|
||||
mcp.WithNumber("child_id", mcp.Required(), mcp.Description("Child thread ID")),
|
||||
mcp.WithString("relation", mcp.Description("Relation: spawned (default), merged_into, references, supersedes")),
|
||||
)
|
||||
|
||||
s.AddTool(threadRelateTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
parentID := int64(clampInt(req, "parent_id", 0, 1, 999999))
|
||||
childID := int64(clampInt(req, "child_id", 0, 1, 999999))
|
||||
relation := getString(req, "relation", "spawned")
|
||||
if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Related thread #%d -> #%d [%s]", parentID, childID, relation)), nil
|
||||
})
|
||||
|
||||
// ─── knox_thread_golden ─────────────────────────────────────
|
||||
goldenTool := mcp.NewTool("knox_thread_golden",
|
||||
mcp.WithDescription("Set, get, or clear the golden thread. The golden thread represents your current focus — new observations auto-link to it."),
|
||||
mcp.WithNumber("thread_id", mcp.Description("Thread ID to set as golden. Omit to query current. Pass 0 to clear.")),
|
||||
)
|
||||
|
||||
s.AddTool(goldenTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
threadID := int64(clampInt(req, "thread_id", 0, 0, 999999))
|
||||
|
||||
if threadID == 0 {
|
||||
// Query or clear
|
||||
current, _ := kdb.GoldenThreadID()
|
||||
if current == 0 {
|
||||
return mcp.NewToolResultText("No golden thread set."), nil
|
||||
}
|
||||
// If thread_id was explicitly 0 and a golden exists, clear it
|
||||
if _, ok := req.Params.Arguments["thread_id"]; ok {
|
||||
kdb.SetGoldenThread(0)
|
||||
t, _ := kdb.GetThread(current)
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Golden thread cleared (was #%d: %s)", current, t.Title)), nil
|
||||
}
|
||||
t, _ := kdb.GetThread(current)
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Golden thread: #%d %s [%s]\n %s", t.ID, t.Title, t.Status, t.Motivation)), nil
|
||||
}
|
||||
|
||||
if err := kdb.SetGoldenThread(threadID); err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
t, _ := kdb.GetThread(threadID)
|
||||
return mcp.NewToolResultText(fmt.Sprintf("Golden thread set to #%d: %s", threadID, t.Title)), nil
|
||||
})
|
||||
|
||||
// ─── knox_topics ───────────────────────────────────────────
|
||||
topicsTool := mcp.NewTool("knox_topics",
|
||||
mcp.WithDescription("List auto-detected topic clusters from TF-IDF analysis. Shows what topics appear across all sources."),
|
||||
mcp.WithNumber("limit", mcp.Description("Max topics (default 10)")),
|
||||
)
|
||||
|
||||
s.AddTool(topicsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
limit := clampInt(req, "limit", 10, 1, 30)
|
||||
entries, _ := kdb.RecentEntries(2000)
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, limit)
|
||||
|
||||
if len(clusters) == 0 {
|
||||
return mcp.NewToolResultText("No topic clusters detected."), nil
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, c := range clusters {
|
||||
total := 0
|
||||
for _, v := range c.BySource {
|
||||
total += v
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("%d. %s (%d entries)\n", i+1, c.Name, total))
|
||||
var sources []string
|
||||
for src, count := range c.BySource {
|
||||
sources = append(sources, fmt.Sprintf("%s:%d", src, count))
|
||||
}
|
||||
b.WriteString(fmt.Sprintf(" Sources: %s\n", strings.Join(sources, ", ")))
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_gaps ─────────────────────────────────────────────
|
||||
gapsTool := mcp.NewTool("knox_gaps",
|
||||
mcp.WithDescription("Find topics with browser activity but no skill coverage. Identifies candidates for new skill creation."),
|
||||
)
|
||||
|
||||
s.AddTool(gapsTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
entries, _ := kdb.RecentEntries(2000)
|
||||
|
||||
skillKeywords := make(map[string]bool)
|
||||
for _, e := range entries {
|
||||
if e.SourceID == "skills-catalog" {
|
||||
for _, w := range index.Tokenize(e.Title + " " + e.Summary) {
|
||||
skillKeywords[w] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, 15)
|
||||
gaps := tfidf.Gaps(clusters, skillKeywords)
|
||||
|
||||
if len(gaps) == 0 {
|
||||
return mcp.NewToolResultText("No gaps found."), nil
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("Found %d topics with browser activity but NO skill coverage:\n\n", len(gaps)))
|
||||
for i, g := range gaps {
|
||||
browserCount := g.BySource["browser-history"]
|
||||
b.WriteString(fmt.Sprintf("%d. %s — %d browser visits\n", i+1, g.Name, browserCount))
|
||||
}
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
// ─── knox_git_status ───────────────────────────────────────
|
||||
gitStatusTool := mcp.NewTool("knox_git_status",
|
||||
mcp.WithDescription("List local git repository status captured by Knox (branch, clean/dirty, ahead/behind, HEAD, last commit). Pass project to filter."),
|
||||
mcp.WithString("project", mcp.Description("Optional project/repo name substring filter")),
|
||||
)
|
||||
|
||||
s.AddTool(gitStatusTool, func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
entries, err := kdb.EntriesBySource("git")
|
||||
if err != nil {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
filter, _ := req.Params.Arguments["project"].(string)
|
||||
|
||||
if len(entries) == 0 {
|
||||
return mcp.NewToolResultText("No git status entries yet. Run `knox git` or wait for the daemon's git timer."), nil
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
shown := 0
|
||||
for _, e := range entries {
|
||||
if filter != "" && !strings.Contains(strings.ToLower(e.Project), strings.ToLower(filter)) {
|
||||
continue
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("%-24s %s\n", e.Project, e.Summary))
|
||||
shown++
|
||||
}
|
||||
b.WriteString(fmt.Sprintf("\n%d / %d repos", shown, len(entries)))
|
||||
return mcp.NewToolResultText(b.String()), nil
|
||||
})
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
// ─── Format helpers ──────────────────────────────────────────
|
||||
|
||||
func formatEntries(entries []db.Entry, detail detailLevel, kdb *db.KnoxDB) string {
|
||||
switch detail {
|
||||
case detailBrief:
|
||||
return formatEntriesBrief(entries)
|
||||
case detailFull:
|
||||
var b strings.Builder
|
||||
for i, e := range entries {
|
||||
if i > 0 {
|
||||
b.WriteString("---\n")
|
||||
}
|
||||
b.WriteString(formatEntryFull(&e))
|
||||
}
|
||||
return b.String()
|
||||
default:
|
||||
var b strings.Builder
|
||||
for _, e := range entries {
|
||||
b.WriteString(formatEntryNormal(&e))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
}
|
||||
|
||||
func formatEntriesBrief(entries []db.Entry) string {
|
||||
var b strings.Builder
|
||||
for _, e := range entries {
|
||||
b.WriteString(fmt.Sprintf("%-8s | %-40s | %-14s | %s\n",
|
||||
shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func formatEntryNormal(e *db.Entry) string {
|
||||
return fmt.Sprintf("%-8s %-40s [%-14s %-4s] seen:%d conf:%.1f\n %s",
|
||||
shortFP(e.Fingerprint), truncateStr(e.Title, 38), e.SourceID, e.Project,
|
||||
e.RefCount, e.LastConfidence, truncateStr(e.Summary, 90))
|
||||
}
|
||||
|
||||
func formatEntryFull(e *db.Entry) string {
|
||||
return fmt.Sprintf(`fingerprint: %s
|
||||
title: %s
|
||||
source: %s
|
||||
path: %s
|
||||
project: %s
|
||||
type: %s
|
||||
first_seen: %s
|
||||
last_seen: %s
|
||||
ref_count: %d
|
||||
confidence: %.1f
|
||||
summary: %s`,
|
||||
e.Fingerprint, e.Title, e.SourceID, e.SourcePath, e.Project, e.ContentType,
|
||||
e.FirstSeen, e.LastSeen, e.RefCount, e.LastConfidence, e.Summary)
|
||||
}
|
||||
|
||||
// ─── Scope filtering ─────────────────────────────────────────
|
||||
|
||||
func filterByScope(entries []db.Entry, scope string) []db.Entry {
|
||||
switch strings.ToLower(scope) {
|
||||
case "skills":
|
||||
return filterSource(entries, "skills-catalog")
|
||||
case "sessions":
|
||||
return filterSource(entries, "opencode-session")
|
||||
case "logs":
|
||||
return filterSource(entries, "opencode-log")
|
||||
default:
|
||||
return entries
|
||||
}
|
||||
}
|
||||
|
||||
func filterSource(entries []db.Entry, sourceID string) []db.Entry {
|
||||
var filtered []db.Entry
|
||||
for _, e := range entries {
|
||||
if e.SourceID == sourceID {
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// ─── Argument helpers ────────────────────────────────────────
|
||||
|
||||
func getString(req mcp.CallToolRequest, key, def string) string {
|
||||
if v, ok := req.Params.Arguments[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func clampInt(req mcp.CallToolRequest, key string, def, min, max int) int {
|
||||
if v, ok := req.Params.Arguments[key].(float64); ok {
|
||||
n := int(v)
|
||||
if n < min {
|
||||
return min
|
||||
}
|
||||
if n > max {
|
||||
return max
|
||||
}
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func shortFP(fp string) string {
|
||||
if len(fp) > 8 {
|
||||
return fp[:8]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func truncateStr(s string, n int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return string(runes[:n]) + "..."
|
||||
}
|
||||
|
||||
func errorResult(msg string) *mcp.CallToolResult {
|
||||
return &mcp.CallToolResult{
|
||||
IsError: true,
|
||||
Content: []mcp.Content{mcp.NewTextContent(msg)},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
)
|
||||
|
||||
func NewObsidianCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var vault string
|
||||
cmd := &cobra.Command{
|
||||
Use: "obsidian",
|
||||
Short: "Ingest Obsidian notes into the knowledge index",
|
||||
Long: `Scans an Obsidian vault for markdown notes, extracts frontmatter
|
||||
(title, tags, dates), and fingerprints them into the index.
|
||||
|
||||
Auto-detects vault path from ~/.config/obsidian/obsidian.json
|
||||
or override with --vault.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
ing := ingest.NewObsidianIngester(vault)
|
||||
log.Printf("[knox] ingesting Obsidian notes...")
|
||||
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
return fmt.Errorf("obsidian ingest: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
fmt.Println("No Obsidian notes found.")
|
||||
return nil
|
||||
}
|
||||
|
||||
var newCount, dupCount int
|
||||
for _, result := range results {
|
||||
_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
LineStart: result.LineStart,
|
||||
LineEnd: result.LineEnd,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "obsidian_ingest",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
} else {
|
||||
dupCount++
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Obsidian notes: %d new, %d updated\n", newCount, dupCount)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&vault, "vault", "v", "", "Obsidian vault path (auto-detect if empty)")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/index"
|
||||
)
|
||||
|
||||
func paginate(entries []db.Entry, page, limit int) []db.Entry {
|
||||
start := (page - 1) * limit
|
||||
if start >= len(entries) {
|
||||
return nil
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(entries) {
|
||||
end = len(entries)
|
||||
}
|
||||
return entries[start:end]
|
||||
}
|
||||
|
||||
func NewStatusCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var jsonOut bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Show knowledge index statistics",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
stats, err := kdb.Stats()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if jsonOut {
|
||||
b, _ := json.MarshalIndent(stats, "", " ")
|
||||
fmt.Println(string(b))
|
||||
return nil
|
||||
}
|
||||
fmt.Println("Knox Knowledge Index")
|
||||
fmt.Println("====================")
|
||||
for k, v := range stats {
|
||||
switch val := v.(type) {
|
||||
case int:
|
||||
fmt.Printf(" %-24s %d\n", k+":", val)
|
||||
case string:
|
||||
fmt.Printf(" %-24s %s\n", k+":", val)
|
||||
default:
|
||||
fmt.Printf(" %-24s %v\n", k+":", val)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&jsonOut, "json", false, "Output as JSON")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func NewQueryCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var page, limit int
|
||||
cmd := &cobra.Command{
|
||||
Use: "query [search text]",
|
||||
Short: "Search the knowledge index",
|
||||
Long: "Search the knowledge index. Reads from stdin pipe if no positional arg given.",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
query := ""
|
||||
if len(args) > 0 {
|
||||
query = args[0]
|
||||
} else {
|
||||
stdin, _ := os.ReadFile(os.Stdin.Name())
|
||||
query = strings.TrimSpace(string(stdin))
|
||||
}
|
||||
if query == "" {
|
||||
return fmt.Errorf("search query required (positional arg or stdin pipe)")
|
||||
}
|
||||
results, err := kdb.Search(query, limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
results = paginate(results, page, limit)
|
||||
if len(results) == 0 {
|
||||
fmt.Println("No results found.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Found %d results for %q (page %d, %d per page):\n\n", len(results), query, page, limit)
|
||||
for _, r := range results {
|
||||
fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(r.Fingerprint), truncateStr(r.Title, 30), r.Project, r.SourceID)
|
||||
if r.Summary != "" {
|
||||
fmt.Printf(" %-8s %s\n", "", truncateStr(r.Summary, 80))
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)")
|
||||
cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func NewRecentCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var page, limit int
|
||||
cmd := &cobra.Command{
|
||||
Use: "recent",
|
||||
Short: "Show recent knowledge entries",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
all, err := kdb.RecentEntries(limit * 10)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries := paginate(all, page, limit)
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("No entries yet. Run `knox watch` to start ingesting.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Recent %d entries (page %d, %d per page):\n\n", len(entries), page, limit)
|
||||
for _, e := range entries {
|
||||
fmt.Printf(" %-8s %-30s [%s] %s\n", shortFP(e.Fingerprint), truncateStr(e.Title, 30), e.Project, e.SourceID)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().IntVarP(&page, "page", "p", 1, "Page number (1-based)")
|
||||
cmd.Flags().IntVarP(&limit, "limit", "l", 20, "Results per page")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func NewReflectCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "reflect",
|
||||
Short: "Process pending sessions for reflection",
|
||||
Long: "Scans for sessions not yet indexed, runs reflection, and reports findings.",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
reflector := index.NewReflector(kdb)
|
||||
results, err := reflector.Reflect()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
fmt.Println("All caught up — no pending sessions to reflect on.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Reflected %d sessions:\n\n", len(results))
|
||||
fmt.Print(reflector.FormatResult(results))
|
||||
|
||||
gaps, _ := reflector.Gaps()
|
||||
if len(gaps) > 0 {
|
||||
fmt.Printf("\nProjects with indexed knowledge:\n")
|
||||
for _, g := range gaps {
|
||||
fmt.Printf(" %-20s %d entries\n", g.Project, g.TotalEntries)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getHomeDir() (string, error) {
|
||||
return os.UserHomeDir()
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/watch"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func NewThreadCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "thread",
|
||||
Short: "Manage knowledge development threads (motivation/intent tracking)",
|
||||
Long: `Threads capture WHY observations were collected — the intent,
|
||||
motivation, and in-flight knowledge development work.
|
||||
|
||||
Subcommands: create, list, link, link-entry, relate, golden, prune, rescan, close, show, note`,
|
||||
}
|
||||
cmd.AddCommand(newThreadCreateCmd(kdb))
|
||||
cmd.AddCommand(newThreadAutoCmd(kdb))
|
||||
cmd.AddCommand(newThreadDraftCmd(kdb))
|
||||
cmd.AddCommand(newThreadUpdateCmd(kdb))
|
||||
cmd.AddCommand(newThreadListCmd(kdb))
|
||||
cmd.AddCommand(newThreadLinkCmd(kdb))
|
||||
cmd.AddCommand(newThreadCloseCmd(kdb))
|
||||
cmd.AddCommand(newThreadShowCmd(kdb))
|
||||
cmd.AddCommand(newThreadNoteCmd(kdb))
|
||||
cmd.AddCommand(newThreadLinkEntryCmd(kdb))
|
||||
cmd.AddCommand(newThreadRelateCmd(kdb))
|
||||
cmd.AddCommand(newThreadGoldenCmd(kdb))
|
||||
cmd.AddCommand(newThreadPruneCmd(kdb))
|
||||
cmd.AddCommand(newThreadRescanCmd(kdb))
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadCreateCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var motivation, priority, tags, provenance string
|
||||
cmd := &cobra.Command{
|
||||
Use: "create <title>",
|
||||
Short: "Create a new knowledge development thread",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
title := strings.Join(args, " ")
|
||||
id, err := kdb.CreateThread(title, motivation, priority, tags, provenance)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Created thread #%d: %s\n", id, title)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "Why this thread exists (the 'so what')")
|
||||
cmd.Flags().StringVarP(&priority, "priority", "p", "medium", "Priority: high, medium, low")
|
||||
cmd.Flags().StringVarP(&tags, "tags", "t", "", "Comma-separated tags")
|
||||
cmd.Flags().StringVarP(&provenance, "provenance", "", "{}", "JSON provenance metadata")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadAutoCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var dryRun bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "auto",
|
||||
Short: "Auto-create threads from high-intent topic clusters (heuristic, no LLM)",
|
||||
Long: `Clusters recent observations with TF-IDF and promotes clusters that cross
|
||||
the "intent bar" into threads, auto-linking their observations.
|
||||
|
||||
The intent bar (tunable via KNOX_THREAD_* env vars) requires: enough distinct
|
||||
sources, at least one non-passive work source (git/gitea/session/obsidian), and
|
||||
observations observed recently. Use --dry-run to preview without writing.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
threader := watch.NewAutoThreader(kdb)
|
||||
threader.DryRun = dryRun
|
||||
created, linked, err := threader.AutoThread()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("\nAuto-thread: %d created, %d observations linked\n", created, linked)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&dryRun, "dry-run", dryRun, "Preview candidate threads without writing")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadDraftCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "draft",
|
||||
Short: "List auto-threaded candidates for LLM title/motivation polish",
|
||||
Long: `Shows threads created by the heuristic auto-threader ('auto_thread'
|
||||
provenance) alongside their linked observations, so an LLM or editor can write
|
||||
a better title and motivation. Apply changes with ` + "`knox thread update <id>`" + `.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
threads, err := kdb.ListThreads("")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var shown int
|
||||
for _, t := range threads {
|
||||
if !isAutoThreaded(t) {
|
||||
continue
|
||||
}
|
||||
obs, _ := kdb.ThreadObservations(t.ID)
|
||||
fmt.Printf("Thread #%d [%s] %d obs\n", t.ID, t.Priority, t.EntryCount)
|
||||
fmt.Printf(" title: %s\n", t.Title)
|
||||
fmt.Printf(" motivation: %s\n", t.Motivation)
|
||||
if t.Tags != "[]" {
|
||||
fmt.Printf(" tags: %s\n", t.Tags)
|
||||
}
|
||||
fmt.Println(" linked observations:")
|
||||
for _, o := range obs {
|
||||
title := truncateStr(o.Title, 70)
|
||||
if o.Summary != "" {
|
||||
title += " — " + truncateStr(o.Summary, 60)
|
||||
}
|
||||
fmt.Printf(" • [%s] %s\n", o.SourceID, title)
|
||||
}
|
||||
fmt.Println()
|
||||
shown++
|
||||
}
|
||||
if shown == 0 {
|
||||
fmt.Println("No auto-threaded candidates found. Run `knox thread auto` (or wait for the watch tick) first.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func isAutoThreaded(t db.Thread) bool {
|
||||
if strings.Contains(t.Provenance, "auto_thread") {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(t.Motivation, "Auto-detected")
|
||||
}
|
||||
|
||||
func newThreadUpdateCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var title, motivation, priority, tags string
|
||||
cmd := &cobra.Command{
|
||||
Use: "update <thread-id>",
|
||||
Short: "Rewrite a thread's title/motivation/priority/tags",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
changed, err := kdb.UpdateThread(id, title, motivation, priority, tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !changed {
|
||||
fmt.Println("No changes applied.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Updated thread #%d\n", id)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&title, "title", "t", "", "New title")
|
||||
cmd.Flags().StringVarP(&motivation, "motivation", "m", "", "New motivation")
|
||||
cmd.Flags().StringVarP(&priority, "priority", "p", "", "New priority (high/medium/low)")
|
||||
cmd.Flags().StringVar(&tags, "tags", "", "New space-separated tags")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadListCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var status string
|
||||
cmd := &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List knowledge development threads",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
threads, err := kdb.ListThreads(status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(threads) == 0 {
|
||||
fmt.Println("No threads. Create one with `knox thread create`.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%-4s %-40s %-10s %-8s %s\n", "ID", "Title", "Status", "Priority", "Entries")
|
||||
fmt.Println(strings.Repeat("-", 75))
|
||||
for _, t := range threads {
|
||||
fmt.Printf("%-4d %-40s %-10s %-8s %d\n", t.ID, truncateStr(t.Title, 38), t.Status, t.Priority, t.EntryCount)
|
||||
if t.Motivation != "" {
|
||||
fmt.Printf(" %s\n", truncateStr(t.Motivation, 70))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&status, "status", "s", "", "Filter by status (active, stalled, resolved)")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadLinkCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var relevance string
|
||||
cmd := &cobra.Command{
|
||||
Use: "link <thread-id> <observation-id>",
|
||||
Short: "Link an observation to a thread",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
threadID, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
obsID, _ := strconv.ParseInt(args[1], 10, 64)
|
||||
if threadID == 0 || obsID == 0 {
|
||||
return fmt.Errorf("thread-id and observation-id must be numeric")
|
||||
}
|
||||
if err := kdb.LinkObservationToThread(threadID, obsID, relevance); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Linked observation #%d to thread #%d\n", obsID, threadID)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&relevance, "relevance", "r", "", "Why this observation matters to the thread")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadCloseCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "close <thread-id>",
|
||||
Short: "Mark a thread as resolved",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
thread, _ := kdb.GetThread(id)
|
||||
if thread == nil {
|
||||
return fmt.Errorf("thread #%d not found", id)
|
||||
}
|
||||
if err := kdb.CloseThread(id); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Closed thread #%d: %s\n", id, thread.Title)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newThreadShowCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "show <thread-id>",
|
||||
Short: "Show thread details with provenance, linked observations, and notes",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
t, err := kdb.GetThread(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t == nil {
|
||||
return fmt.Errorf("thread #%d not found", id)
|
||||
}
|
||||
goldenID, _ := kdb.GoldenThreadID()
|
||||
fmt.Printf("Thread #%d: %s", t.ID, t.Title)
|
||||
if goldenID == id {
|
||||
fmt.Print(" [GOLDEN]")
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Printf(" Status: %s\n", t.Status)
|
||||
fmt.Printf(" Priority: %s\n", t.Priority)
|
||||
fmt.Printf(" Created: %s\n", t.CreatedAt)
|
||||
fmt.Printf(" Updated: %s\n", t.UpdatedAt)
|
||||
if t.ResolvedAt != "" {
|
||||
fmt.Printf(" Resolved: %s\n", t.ResolvedAt)
|
||||
}
|
||||
if t.Motivation != "" {
|
||||
fmt.Printf(" Motivation: %s\n", t.Motivation)
|
||||
}
|
||||
if t.Tags != "[]" {
|
||||
fmt.Printf(" Tags: %s\n", t.Tags)
|
||||
}
|
||||
if t.Provenance != "{}" {
|
||||
fmt.Printf(" Provenance: %s\n", t.Provenance)
|
||||
}
|
||||
obs, _ := kdb.ThreadObservations(id)
|
||||
if len(obs) > 0 {
|
||||
fmt.Printf("\nLinked observations (%d):\n", len(obs))
|
||||
for _, o := range obs {
|
||||
fmt.Printf(" #%-4d %-30s [%s] trigger=%s\n", o.ID, truncateStr(o.Title, 28), o.SourceID, o.Trigger)
|
||||
}
|
||||
}
|
||||
provenance, _ := kdb.ThreadProvenance(id)
|
||||
if len(provenance) > 0 {
|
||||
fmt.Printf("\nProvenance graph (%d links):\n", len(provenance))
|
||||
for _, p := range provenance {
|
||||
fmt.Printf(" %s --[%s]--> thread:%d\n", shortFP(p.Fingerprint), p.Relation, id)
|
||||
}
|
||||
}
|
||||
notes, _ := kdb.ThreadNotes(id)
|
||||
if len(notes) > 0 {
|
||||
fmt.Printf("\nNotes (%d):\n", len(notes))
|
||||
for _, n := range notes {
|
||||
fmt.Printf(" [%s] %s\n", n.CreatedAt, n.Note)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newThreadLinkEntryCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var relation string
|
||||
cmd := &cobra.Command{
|
||||
Use: "link-entry <thread-id> <entry-fingerprint>",
|
||||
Short: "Link an entry to a thread via provenance graph",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
if err := kdb.LinkEntryToThread(id, args[1], relation); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Linked entry %s to thread #%d [%s]\n", shortFP(args[1]), id, relation)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&relation, "relation", "r", "produced", "Relation type (produced, references, motivated)")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadRelateCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var relation string
|
||||
cmd := &cobra.Command{
|
||||
Use: "relate <parent-id> <child-id>",
|
||||
Short: "Relate two threads (parent spawned child, etc.)",
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
parentID, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
childID, _ := strconv.ParseInt(args[1], 10, 64)
|
||||
if parentID == 0 || childID == 0 {
|
||||
return fmt.Errorf("both thread IDs must be numeric")
|
||||
}
|
||||
if err := kdb.LinkEntryToThread(childID, db.ThreadFP(parentID), relation); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Related thread #%d -> #%d [%s]\n", parentID, childID, relation)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVarP(&relation, "relation", "r", "spawned", "Relation type (spawned, merged_into, references, supersedes)")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadGoldenCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var unset bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "golden [thread-id]",
|
||||
Short: "Set/show/clear the golden thread (current focus). Auto-links new observations.",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
if unset {
|
||||
kdb.SetGoldenThread(0)
|
||||
fmt.Println("Golden thread cleared.")
|
||||
return nil
|
||||
}
|
||||
if len(args) > 0 {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
if err := kdb.SetGoldenThread(id); err != nil {
|
||||
return err
|
||||
}
|
||||
t, _ := kdb.GetThread(id)
|
||||
if t != nil {
|
||||
fmt.Printf("Golden thread set to #%d: %s\n", id, t.Title)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
goldenID, _ := kdb.GoldenThreadID()
|
||||
if goldenID == 0 {
|
||||
fmt.Println("No golden thread set.")
|
||||
return nil
|
||||
}
|
||||
t, _ := kdb.GetThread(goldenID)
|
||||
if t != nil {
|
||||
fmt.Printf("Golden thread: #%d %s [%s/%s]\n %s\n", t.ID, t.Title, t.Status, t.Priority, t.Motivation)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&unset, "unset", false, "Clear the golden thread")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newThreadNoteCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "note <thread-id> <text>",
|
||||
Short: "Add a note to a thread",
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
note := strings.Join(args[1:], " ")
|
||||
nid, err := kdb.AddThreadNote(id, note)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Added note #%d to thread #%d\n", nid, id)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newThreadPruneCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "prune <thread-id>",
|
||||
Short: "Remove irrelevant auto-linked observations from a thread",
|
||||
Long: `Scans auto-linked observations and removes ones that don't share keywords with the thread's title, motivation, or tags.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
t, err := kdb.GetThread(id)
|
||||
if err != nil || t == nil {
|
||||
return fmt.Errorf("thread #%d not found", id)
|
||||
}
|
||||
keywords, bigrams, err := kdb.ThreadRelevanceProfile(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(keywords) == 0 {
|
||||
return fmt.Errorf("no significant keywords found in thread")
|
||||
}
|
||||
count, err := kdb.PruneThread(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Pruned %d irrelevant links from thread #%d (%s)\n", count, id, t.Title)
|
||||
fmt.Printf(" Kept observations matching: %v (bigrams: %v)\n", keywords, bigrams)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newThreadRescanCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "rescan <thread-id>",
|
||||
Short: "Link unlinked observations that pass the thread's relevance profile",
|
||||
Long: `Scans all observations and links ones matching the thread's DF-filtered keyword profile. Inverse of prune.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
id, _ := strconv.ParseInt(args[0], 10, 64)
|
||||
if id == 0 {
|
||||
return fmt.Errorf("thread-id must be numeric")
|
||||
}
|
||||
t, err := kdb.GetThread(id)
|
||||
if err != nil || t == nil {
|
||||
return fmt.Errorf("thread #%d not found", id)
|
||||
}
|
||||
count, err := kdb.RescanThread(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Linked %d observations to thread #%d (%s)\n", count, id, t.Title)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/index"
|
||||
)
|
||||
|
||||
func NewTopicsCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "topics",
|
||||
Short: "Auto-detect topic clusters from TF-IDF analysis of the knowledge index",
|
||||
Long: `Scans all entries, computes TF-IDF term scores, and clusters related
|
||||
observations into topics. Shows gap analysis for browser-researched
|
||||
topics not covered by skills.
|
||||
|
||||
Subcommands: list, gaps`,
|
||||
}
|
||||
|
||||
cmd.AddCommand(newTopicListCmd(kdb))
|
||||
cmd.AddCommand(newTopicGapsCmd(kdb))
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newTopicListCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "list",
|
||||
Short: "List auto-detected topic clusters",
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
entries, err := kdb.RecentEntries(2000)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, 15)
|
||||
|
||||
if len(clusters) == 0 {
|
||||
fmt.Println("No topic clusters detected.")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%-3s %-50s %-8s %s\n", "ID", "Topic", "Entries", "Sources")
|
||||
fmt.Println(strings.Repeat("-", 75))
|
||||
for i, c := range clusters {
|
||||
total := 0
|
||||
for _, v := range c.BySource {
|
||||
total += v
|
||||
}
|
||||
sources := ""
|
||||
for src, count := range c.BySource {
|
||||
if sources != "" {
|
||||
sources += ", "
|
||||
}
|
||||
sources += fmt.Sprintf("%s:%d", shortSource(src), count)
|
||||
}
|
||||
fmt.Printf("%-3d %-50s %-8d %s\n", i+1, truncateStr(c.Name, 48), total, sources)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newTopicGapsCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "gaps",
|
||||
Short: "Show topics with browser activity but no skill coverage",
|
||||
Long: `Cross-references browser history against the skills catalog.
|
||||
Flags topics that have significant browser research but zero skill entries.
|
||||
|
||||
These are candidates for new skill creation.`,
|
||||
RunE: func(c *cobra.Command, args []string) error {
|
||||
entries, err := kdb.RecentEntries(2000)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Build skill keyword set
|
||||
skillKeywords := make(map[string]bool)
|
||||
for _, e := range entries {
|
||||
if e.SourceID == "skills-catalog" {
|
||||
for _, w := range tokenize(e.Title + " " + e.Summary) {
|
||||
skillKeywords[w] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, 15)
|
||||
gaps := tfidf.Gaps(clusters, skillKeywords)
|
||||
|
||||
if len(gaps) == 0 {
|
||||
fmt.Println("No gaps found — all browser-researched topics have skill coverage.")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("\nFound %d topics with browser activity but NO skill coverage:\n\n", len(gaps))
|
||||
for i, g := range gaps {
|
||||
browserCount := g.BySource["browser-history"]
|
||||
fmt.Printf("%d. %s\n", i+1, g.Name)
|
||||
fmt.Printf(" %d browser visits · keywords: %v\n", browserCount, g.Keywords)
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
fmt.Println("Consider creating skills for these topics with `knox asses` or via opencode.")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func shortSource(s string) string {
|
||||
switch s {
|
||||
case "opencode-session":
|
||||
return "session"
|
||||
case "skills-catalog":
|
||||
return "skill"
|
||||
case "browser-history":
|
||||
return "browser"
|
||||
case "obsidian":
|
||||
return "notes"
|
||||
case "opencode-log":
|
||||
return "logs"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func tokenize(text string) []string {
|
||||
re := regexp.MustCompile(`[a-z]{4,}`)
|
||||
return re.FindAllString(strings.ToLower(text), -1)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/watch"
|
||||
)
|
||||
|
||||
func NewWatchCmd(kdb *db.KnoxDB) *cobra.Command {
|
||||
var dirs []string
|
||||
var quiet bool
|
||||
cmd := &cobra.Command{
|
||||
Use: "watch",
|
||||
Short: "Watch directories for new session data and ingest into index",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if len(dirs) == 0 {
|
||||
dirs = defaultWatchDirs()
|
||||
}
|
||||
if quiet {
|
||||
log.SetOutput(io.Discard)
|
||||
}
|
||||
log.Printf("[knox] starting watcher, scanning %d directories", len(dirs))
|
||||
w := watch.New(kdb, dirs)
|
||||
if err := w.Start(); err != nil {
|
||||
log.Fatalf("watcher error: %v", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to watch (default: opencode storage/log + skills)")
|
||||
cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Suppress per-entry log output")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func defaultWatchDirs() []string {
|
||||
home, _ := getHomeDir()
|
||||
return []string{
|
||||
home + "/.local/share/opencode/storage/session_diff",
|
||||
home + "/.local/share/opencode/log",
|
||||
home + "/.skills",
|
||||
}
|
||||
}
|
||||
+1325
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
package db
|
||||
|
||||
const Schema = `
|
||||
-- IMMUTABLE LOG: every observation is append-only, never mutated
|
||||
CREATE TABLE IF NOT EXISTS observations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
fingerprint TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL,
|
||||
source_path TEXT,
|
||||
project TEXT,
|
||||
content_type TEXT,
|
||||
title TEXT,
|
||||
summary TEXT,
|
||||
collected_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
created_at TEXT,
|
||||
line_start INTEGER,
|
||||
line_end INTEGER,
|
||||
confidence REAL DEFAULT 0.5,
|
||||
ingester_version TEXT DEFAULT '1',
|
||||
trigger TEXT,
|
||||
provenance TEXT DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_fingerprint ON observations(fingerprint);
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_collected ON observations(collected_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_created ON observations(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_source ON observations(source_id, source_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_project ON observations(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_obs_trigger ON observations(trigger);
|
||||
|
||||
-- MATERIALIZED CACHE: current state, reconstructable from observations
|
||||
CREATE TABLE IF NOT EXISTS entries (
|
||||
fingerprint TEXT PRIMARY KEY,
|
||||
source_id TEXT NOT NULL,
|
||||
source_path TEXT,
|
||||
project TEXT,
|
||||
content_type TEXT,
|
||||
title TEXT,
|
||||
summary TEXT,
|
||||
first_seen TEXT,
|
||||
last_seen TEXT,
|
||||
ref_count INTEGER DEFAULT 1,
|
||||
last_confidence REAL DEFAULT 0.5
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_entries_project ON entries(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_entries_seen ON entries(last_seen);
|
||||
|
||||
-- PROVENANCE LINKS: multi-hop lineage between entries
|
||||
CREATE TABLE IF NOT EXISTS provenance_links (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
descendant_fp TEXT NOT NULL,
|
||||
ancestor_fp TEXT NOT NULL,
|
||||
hop_distance INTEGER NOT NULL,
|
||||
relation TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pl_descendant ON provenance_links(descendant_fp);
|
||||
CREATE INDEX IF NOT EXISTS idx_pl_ancestor ON provenance_links(ancestor_fp);
|
||||
|
||||
-- REFLECTION STATE: per-session tracking
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
project TEXT,
|
||||
title TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
agent_count INTEGER DEFAULT 0,
|
||||
started_at TEXT,
|
||||
ended_at TEXT,
|
||||
last_seen TEXT DEFAULT (datetime('now')),
|
||||
indexed INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- SOURCE REGISTRY
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
last_event TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-session', 'Opencode Session Diffs');
|
||||
INSERT OR IGNORE INTO sources (id, name) VALUES ('opencode-log', 'Opencode Log Files');
|
||||
INSERT OR IGNORE INTO sources (id, name) VALUES ('git', 'Git History');
|
||||
INSERT OR IGNORE INTO sources (id, name) VALUES ('filesystem', 'File System Watcher');
|
||||
|
||||
-- KNOWLEDGE THREADS: why observations exist, what questions they answer
|
||||
CREATE TABLE IF NOT EXISTS threads (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
motivation TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
priority TEXT DEFAULT 'medium',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
resolved_at TEXT,
|
||||
tags TEXT DEFAULT '[]',
|
||||
provenance TEXT DEFAULT '{}'
|
||||
);
|
||||
|
||||
-- SETTINGS: key-value store for runtime configuration
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_threads_status ON threads(status);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS thread_observations (
|
||||
thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE,
|
||||
observation_id INTEGER NOT NULL REFERENCES observations(id) ON DELETE CASCADE,
|
||||
relevance TEXT,
|
||||
added_at TEXT DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (thread_id, observation_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS thread_notes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
thread_id INTEGER NOT NULL REFERENCES threads(id) ON DELETE CASCADE,
|
||||
note TEXT NOT NULL,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- TOPICS: TF-IDF clustered groups of related observations
|
||||
CREATE TABLE IF NOT EXISTS topics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
keywords TEXT,
|
||||
score REAL DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
`
|
||||
@@ -0,0 +1,100 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
)
|
||||
|
||||
type Reflector struct {
|
||||
db *db.KnoxDB
|
||||
}
|
||||
|
||||
func NewReflector(kdb *db.KnoxDB) *Reflector {
|
||||
return &Reflector{db: kdb}
|
||||
}
|
||||
|
||||
type ReflectionResult struct {
|
||||
SessionID string
|
||||
Project string
|
||||
EntryCount int
|
||||
Status string
|
||||
}
|
||||
|
||||
func (r *Reflector) Reflect() ([]ReflectionResult, error) {
|
||||
sessions, err := r.db.PendingSessions()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pending sessions: %w", err)
|
||||
}
|
||||
|
||||
var results []ReflectionResult
|
||||
for _, s := range sessions {
|
||||
entries, err := r.db.EntriesByProject(s.Project, 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := ReflectionResult{
|
||||
SessionID: s.SessionID,
|
||||
Project: s.Project,
|
||||
EntryCount: len(entries),
|
||||
Status: "reflected",
|
||||
}
|
||||
|
||||
if err := r.db.MarkSessionIndexed(s.SessionID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type GapReport struct {
|
||||
Project string
|
||||
TotalEntries int
|
||||
}
|
||||
|
||||
func (r *Reflector) Gaps() ([]GapReport, error) {
|
||||
entries, err := r.db.RecentEntries(500)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
projects := make(map[string]int)
|
||||
for _, e := range entries {
|
||||
if e.Project != "" {
|
||||
projects[e.Project]++
|
||||
}
|
||||
}
|
||||
|
||||
var reports []GapReport
|
||||
for proj, count := range projects {
|
||||
reports = append(reports, GapReport{
|
||||
Project: proj,
|
||||
TotalEntries: count,
|
||||
})
|
||||
}
|
||||
return reports, nil
|
||||
}
|
||||
|
||||
func (r *Reflector) FormatResult(results []ReflectionResult) string {
|
||||
var b strings.Builder
|
||||
for _, res := range results {
|
||||
b.WriteString(fmt.Sprintf(" %-12s %-20s %d entries [%s]\n",
|
||||
truncateStr(res.SessionID, 12),
|
||||
res.Project,
|
||||
res.EntryCount,
|
||||
res.Status,
|
||||
))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func truncateStr(s string, n int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return string(runes[:n]) + "..."
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
)
|
||||
|
||||
type TopicCluster struct {
|
||||
Name string
|
||||
Keywords []string
|
||||
Score float64
|
||||
Entries []db.Entry
|
||||
BySource map[string]int
|
||||
}
|
||||
|
||||
var tokenRE = regexp.MustCompile(`[a-z]{4,}`)
|
||||
var hexTokenRE = regexp.MustCompile(`^[a-f0-9]{8,}$`)
|
||||
|
||||
func Tokenize(text string) []string { return tokenize(text) }
|
||||
func tokenize(text string) []string {
|
||||
text = strings.ToLower(text)
|
||||
text = strings.ReplaceAll(text, "-", " ")
|
||||
text = strings.ReplaceAll(text, "_", " ")
|
||||
return tokenRE.FindAllString(text, -1)
|
||||
}
|
||||
|
||||
type document struct {
|
||||
id int
|
||||
title string
|
||||
text string
|
||||
source string
|
||||
entry db.Entry
|
||||
}
|
||||
|
||||
// TF-IDF index built from all observations
|
||||
type TFIDFIndex struct {
|
||||
docCount int
|
||||
docFreq map[string]int // term → how many docs contain it
|
||||
termDocs []map[string]float64 // per-doc: term → TF score
|
||||
documents []document
|
||||
}
|
||||
|
||||
func BuildTFIDF(entries []db.Entry) *TFIDFIndex {
|
||||
idx := &TFIDFIndex{
|
||||
docFreq: make(map[string]int),
|
||||
termDocs: make([]map[string]float64, 0, len(entries)),
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
text := e.Title + " " + e.Summary + " " + e.SourceID + " " + e.Project
|
||||
tokens := tokenize(text)
|
||||
if len(tokens) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
doc := document{
|
||||
id: idx.docCount,
|
||||
title: e.Title,
|
||||
text: text,
|
||||
source: e.SourceID,
|
||||
entry: e,
|
||||
}
|
||||
|
||||
// Term frequency in this doc
|
||||
tf := make(map[string]float64)
|
||||
seen := make(map[string]bool)
|
||||
for _, t := range tokens {
|
||||
if hexTokenRE.MatchString(t) || len(t) > 25 {
|
||||
continue
|
||||
}
|
||||
tf[t]++
|
||||
if !seen[t] {
|
||||
idx.docFreq[t]++
|
||||
seen[t] = true
|
||||
}
|
||||
}
|
||||
// Normalize TF by doc length
|
||||
docLen := float64(len(tokens))
|
||||
for t, c := range tf {
|
||||
tf[t] = c / docLen
|
||||
}
|
||||
|
||||
idx.termDocs = append(idx.termDocs, tf)
|
||||
idx.documents = append(idx.documents, doc)
|
||||
idx.docCount++
|
||||
}
|
||||
|
||||
return idx
|
||||
}
|
||||
|
||||
func (idx *TFIDFIndex) TFIDF(term string, docIdx int) float64 {
|
||||
tf, ok := idx.termDocs[docIdx][term]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
df := idx.docFreq[term]
|
||||
if df == 0 {
|
||||
return 0
|
||||
}
|
||||
idf := math.Log(float64(idx.docCount) / float64(df))
|
||||
return tf * idf
|
||||
}
|
||||
|
||||
// TopTerms returns the N highest TF-IDF scoring terms for a document
|
||||
func (idx *TFIDFIndex) TopTerms(docIdx int, n int) []struct {
|
||||
Term string
|
||||
Score float64
|
||||
} {
|
||||
var scored []struct {
|
||||
Term string
|
||||
Score float64
|
||||
}
|
||||
for t := range idx.termDocs[docIdx] {
|
||||
s := idx.TFIDF(t, docIdx)
|
||||
if s > 0 {
|
||||
scored = append(scored, struct {
|
||||
Term string
|
||||
Score float64
|
||||
}{t, s})
|
||||
}
|
||||
}
|
||||
sort.Slice(scored, func(i, j int) bool { return scored[i].Score > scored[j].Score })
|
||||
if len(scored) > n {
|
||||
scored = scored[:n]
|
||||
}
|
||||
return scored
|
||||
}
|
||||
|
||||
// Cluster groups observations into topics by shared top TF-IDF terms
|
||||
func (idx *TFIDFIndex) Cluster(minSharedTerms int, maxTopics int) []TopicCluster {
|
||||
if maxTopics <= 0 {
|
||||
maxTopics = 20
|
||||
}
|
||||
if minSharedTerms <= 0 {
|
||||
minSharedTerms = 2
|
||||
}
|
||||
|
||||
// Get top 5 terms per doc
|
||||
type docTerms struct {
|
||||
docIdx int
|
||||
terms []string
|
||||
}
|
||||
var docTermList []docTerms
|
||||
for i := 0; i < idx.docCount; i++ {
|
||||
tt := idx.TopTerms(i, 5)
|
||||
if len(tt) >= minSharedTerms {
|
||||
terms := make([]string, len(tt))
|
||||
for j, t := range tt {
|
||||
terms[j] = t.Term
|
||||
}
|
||||
docTermList = append(docTermList, docTerms{i, terms})
|
||||
}
|
||||
}
|
||||
|
||||
// Greedy clustering: docs sharing >= minSharedTerms terms become a topic
|
||||
var clusters []TopicCluster
|
||||
assigned := make(map[int]bool)
|
||||
|
||||
for _, dt := range docTermList {
|
||||
if assigned[dt.docIdx] {
|
||||
continue
|
||||
}
|
||||
|
||||
cluster := TopicCluster{
|
||||
Keywords: dt.terms,
|
||||
Score: 0,
|
||||
BySource: make(map[string]int),
|
||||
}
|
||||
|
||||
// The seed doc belongs to this cluster too.
|
||||
if seed := idx.documents[dt.docIdx]; seed.source != "" {
|
||||
cluster.BySource[seed.source]++
|
||||
cluster.Entries = append(cluster.Entries, seed.entry)
|
||||
}
|
||||
|
||||
// Find all docs sharing terms with this seed
|
||||
for _, other := range docTermList {
|
||||
if assigned[other.docIdx] {
|
||||
continue
|
||||
}
|
||||
shared := 0
|
||||
for _, t1 := range dt.terms {
|
||||
for _, t2 := range other.terms {
|
||||
if t1 == t2 {
|
||||
shared++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if shared >= minSharedTerms {
|
||||
assigned[other.docIdx] = true
|
||||
e := idx.documents[other.docIdx]
|
||||
cluster.BySource[e.source]++
|
||||
cluster.Entries = append(cluster.Entries, e.entry)
|
||||
|
||||
// Score is sum of TF-IDF of shared terms
|
||||
for _, t := range dt.terms {
|
||||
cluster.Score += idx.TFIDF(t, other.docIdx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(cluster.BySource) > 0 {
|
||||
// Name the topic by top scoring TF-IDF terms across all docs
|
||||
allTerms := make(map[string]float64)
|
||||
for _, t := range dt.terms {
|
||||
for i := 0; i < idx.docCount; i++ {
|
||||
allTerms[t] += idx.TFIDF(t, i)
|
||||
}
|
||||
}
|
||||
type kv struct {
|
||||
k string
|
||||
v float64
|
||||
}
|
||||
var sorted []kv
|
||||
for k, v := range allTerms {
|
||||
sorted = append(sorted, kv{k, v})
|
||||
}
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].v > sorted[j].v })
|
||||
|
||||
var nameParts []string
|
||||
for _, kv := range sorted {
|
||||
if len(nameParts) >= 3 {
|
||||
break
|
||||
}
|
||||
nameParts = append(nameParts, kv.k)
|
||||
}
|
||||
cluster.Name = strings.Join(nameParts, " / ")
|
||||
|
||||
clusters = append(clusters, cluster)
|
||||
if len(clusters) >= maxTopics {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return clusters
|
||||
}
|
||||
|
||||
// Gaps finds topics with high browser activity but no skill coverage
|
||||
func (idx *TFIDFIndex) Gaps(clusters []TopicCluster, skillKeywords map[string]bool) []TopicCluster {
|
||||
var gaps []TopicCluster
|
||||
for _, c := range clusters {
|
||||
browserScore := c.BySource["browser-history"]
|
||||
skillScore := c.BySource["skills-catalog"]
|
||||
if browserScore > 0 && skillScore == 0 {
|
||||
gaps = append(gaps, c)
|
||||
}
|
||||
}
|
||||
sort.Slice(gaps, func(i, j int) bool {
|
||||
return gaps[i].BySource["browser-history"] > gaps[j].BySource["browser-history"]
|
||||
})
|
||||
return gaps
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
chromiumHistoryPath = ".config/chromium/Default/History"
|
||||
minHistoryPath = ".config/Min/BrowserHistory.db"
|
||||
)
|
||||
|
||||
// noiseURLPatterns are URLs that represent navigation noise rather than knowledge.
|
||||
var noiseURLPatterns = []string{
|
||||
"opencode.ai",
|
||||
"auth.opencode.ai",
|
||||
"github.com/login/oauth/authorize",
|
||||
"accounts.google.com",
|
||||
"google.com/accounts",
|
||||
"kagi.com",
|
||||
"chatgpt.com",
|
||||
// knox indexing its own web UI is self-referential noise
|
||||
"localhost:8924",
|
||||
"localhost:18925",
|
||||
"127.0.0.1:8924",
|
||||
"127.0.0.1:18925",
|
||||
}
|
||||
|
||||
func noiseURL(url string) bool {
|
||||
for _, p := range noiseURLPatterns {
|
||||
if strings.Contains(url, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type BrowserHistoryIngester struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func NewBrowserHistoryIngester() *BrowserHistoryIngester {
|
||||
return &BrowserHistoryIngester{Version: "browser-history/v1"}
|
||||
}
|
||||
|
||||
func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" }
|
||||
|
||||
type historyRow struct {
|
||||
URL string
|
||||
Title string
|
||||
VisitTime time.Time
|
||||
VisitCount int
|
||||
}
|
||||
|
||||
func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) {
|
||||
home, _ := os.UserHomeDir()
|
||||
var results []*IngestResult
|
||||
|
||||
// Chromium
|
||||
chromiumDB := filepath.Join(home, chromiumHistoryPath)
|
||||
cr, err := b.ingestChromium(chromiumDB)
|
||||
if err == nil {
|
||||
results = append(results, cr...)
|
||||
}
|
||||
|
||||
// Min
|
||||
minDB := filepath.Join(home, minHistoryPath)
|
||||
minResults, err := b.ingestMin(minDB)
|
||||
if err == nil {
|
||||
results = append(results, minResults...)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) {
|
||||
if !fileExists(path) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Copy to avoid lock issues (Chromium uses WAL)
|
||||
tmp := path + ".knox_tmp"
|
||||
if err := copyFile(path, tmp); err != nil {
|
||||
return nil, fmt.Errorf("copy chromium db: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp)
|
||||
|
||||
db, err := sql.Open("sqlite", tmp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open chromium copy: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query(
|
||||
`SELECT url, COALESCE(title,''), last_visit_time, visit_count
|
||||
FROM urls
|
||||
WHERE last_visit_time > 0
|
||||
ORDER BY last_visit_time DESC
|
||||
LIMIT 1000`,
|
||||
)
|
||||
if err != nil {
|
||||
// Table might not exist or schema mismatch
|
||||
return nil, fmt.Errorf("query chromium urls: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []*IngestResult
|
||||
for rows.Next() {
|
||||
var url, title string
|
||||
var visitTimeMicro int64
|
||||
var visitCount int
|
||||
if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Chromium webkit time: microseconds since 1601-01-01 UTC
|
||||
visitTime := webkitToTime(visitTimeMicro)
|
||||
|
||||
if result := b.makeResult(url, title, visitTime, visitCount); result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) {
|
||||
if !fileExists(path) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
tmp := path + ".knox_tmp"
|
||||
if err := copyFile(path, tmp); err != nil {
|
||||
return nil, fmt.Errorf("copy min db: %w", err)
|
||||
}
|
||||
defer os.Remove(tmp)
|
||||
|
||||
db, err := sql.Open("sqlite", tmp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open min copy: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query(
|
||||
`SELECT url, COALESCE(title,''), timestamp
|
||||
FROM history
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1000`,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query min history: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []*IngestResult
|
||||
for rows.Next() {
|
||||
var url, title string
|
||||
var ts int64
|
||||
if err := rows.Scan(&url, &title, &ts); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
visitTime := time.Unix(ts, 0)
|
||||
if result := b.makeResult(url, title, visitTime, 1); result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult {
|
||||
if url == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip chrome://, about://, file://, devtools
|
||||
if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") ||
|
||||
strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") ||
|
||||
strings.HasPrefix(url, "chrome-extension://") {
|
||||
return nil
|
||||
}
|
||||
|
||||
if noiseURL(url) {
|
||||
return nil
|
||||
}
|
||||
|
||||
displayTitle := title
|
||||
if displayTitle == "" {
|
||||
displayTitle = extractDomain(url)
|
||||
}
|
||||
|
||||
fp := Fingerprint([]byte(url))
|
||||
|
||||
confidence := 0.6
|
||||
if visitCount > 5 {
|
||||
confidence = 0.9
|
||||
} else if visitCount > 1 {
|
||||
confidence = 0.7
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: b.SourceID(),
|
||||
SourcePath: url,
|
||||
ContentType: "url",
|
||||
Title: displayTitle,
|
||||
Summary: url,
|
||||
CreatedAt: visitTime.Format(time.RFC3339),
|
||||
Confidence: confidence,
|
||||
IngesterVersion: b.Version,
|
||||
Provenance: map[string]any{
|
||||
"url": url,
|
||||
"visit_count": visitCount,
|
||||
"visited_at": visitTime.Format(time.RFC3339),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time
|
||||
func webkitToTime(micros int64) time.Time {
|
||||
// WebKit epoch: 1601-01-01 UTC
|
||||
// Unix epoch: 1970-01-01 UTC
|
||||
// Difference: 11644473600 seconds
|
||||
secs := micros / 1_000_000
|
||||
if secs < 11644473600 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(secs-11644473600, 0)
|
||||
}
|
||||
|
||||
func extractDomain(url string) string {
|
||||
url = strings.TrimPrefix(url, "https://")
|
||||
url = strings.TrimPrefix(url, "http://")
|
||||
parts := strings.Split(url, "/")
|
||||
if len(parts) > 0 {
|
||||
return parts[0]
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(dst, data, 0644)
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GitIngester tracks project status for local git repositories.
|
||||
//
|
||||
// For each repo under one or more scan roots (default ~/src) it records the
|
||||
// current branch, whether the working tree is clean, ahead/behind counts vs
|
||||
// the upstream, the HEAD hash, and the last commit time + subject. The entry's
|
||||
// CreatedAt is the last commit time, so staleness/dedup works naturally and
|
||||
// an entry updates when new commits land.
|
||||
//
|
||||
// When Recursive is set, repos are discovered at any depth under a root
|
||||
// (following symlinked directories), not just direct children — this picks up
|
||||
// nested repos like ~/assistant/kitmaker or ~/Arduino/*/... Repos already have
|
||||
// a stable identity via their real (symlink-resolved) path, so overlapping
|
||||
// roots across symlinks do not duplicate entries.
|
||||
type GitIngester struct {
|
||||
Version string
|
||||
Root string // single scan root; overridden by Roots (empty => $HOME/src)
|
||||
Roots []string // multi-scan roots; overrides Root when non-empty
|
||||
Recursive bool // recurse into subdirs (and follow dir symlinks) to find nested .git
|
||||
}
|
||||
|
||||
func NewGitIngester() *GitIngester {
|
||||
return &GitIngester{Version: "git/v1"}
|
||||
}
|
||||
|
||||
func (g *GitIngester) SourceID() string { return "git" }
|
||||
|
||||
// scanRoots returns the effective list of scan roots.
|
||||
func (g *GitIngester) scanRoots() []string {
|
||||
if len(g.Roots) > 0 {
|
||||
return g.Roots
|
||||
}
|
||||
if g.Root != "" {
|
||||
return []string{g.Root}
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{filepath.Join(home, "src")}
|
||||
}
|
||||
|
||||
func (g *GitIngester) IngestAll() ([]*IngestResult, error) {
|
||||
// Dedup by real path so overlapping roots / symlinked roots don't double-add.
|
||||
seen := make(map[string]struct{})
|
||||
var results []*IngestResult
|
||||
for _, root := range g.scanRoots() {
|
||||
for _, repoPath := range collectGitRepos(root, g.Recursive) {
|
||||
real, err := filepath.EvalSymlinks(repoPath)
|
||||
if err != nil {
|
||||
real = repoPath
|
||||
}
|
||||
if _, dup := seen[real]; dup {
|
||||
continue
|
||||
}
|
||||
seen[real] = struct{}{}
|
||||
if r := g.repoToResult(real); r != nil {
|
||||
results = append(results, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isGitRepo(path string) bool {
|
||||
info, err := os.Stat(filepath.Join(path, ".git"))
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
|
||||
// collectGitRepos returns paths of git repos under root. In non-recursive mode
|
||||
// only direct children are considered (root itself is not counted), matching
|
||||
// the original behavior. In recursive mode the root is walked to any depth,
|
||||
// descending into symlinked directories, up to and including the root if it is
|
||||
// itself a git repo. Descending stops once a git repo is found, so nested
|
||||
// repos (e.g. a vendored copy inside another repo) are treated as leaves.
|
||||
func collectGitRepos(root string, recursive bool) []string {
|
||||
visited := make(map[string]struct{})
|
||||
var out []string
|
||||
|
||||
var walk func(dir string)
|
||||
walk = func(dir string) {
|
||||
real, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := visited[real]; ok {
|
||||
return
|
||||
}
|
||||
visited[real] = struct{}{}
|
||||
|
||||
if isGitRepo(real) {
|
||||
out = append(out, real)
|
||||
return
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(real)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
child := filepath.Join(real, e.Name())
|
||||
if recursive {
|
||||
walk(child)
|
||||
continue
|
||||
}
|
||||
if e.IsDir() && isGitRepo(child) {
|
||||
out = append(out, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(root)
|
||||
return out
|
||||
}
|
||||
|
||||
type GitStatus struct {
|
||||
Branch string
|
||||
Head string
|
||||
Dirty int
|
||||
Ahead int
|
||||
Behind int
|
||||
LastCommit string
|
||||
Subject string
|
||||
}
|
||||
|
||||
func (g *GitIngester) repoToResult(repoPath string) *IngestResult {
|
||||
status := gitStatus(repoPath)
|
||||
name := filepath.Base(repoPath)
|
||||
fp := Fingerprint([]byte("git:repo:" + repoPath))
|
||||
|
||||
clean := "clean"
|
||||
if status.Dirty > 0 {
|
||||
clean = fmt.Sprintf("%d change(s)", status.Dirty)
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("%s @ %s — %s", name, status.Branch, clean)
|
||||
if status.Ahead > 0 || status.Behind > 0 {
|
||||
summary += fmt.Sprintf(", ahead %d / behind %d", status.Ahead, status.Behind)
|
||||
}
|
||||
if status.Subject != "" {
|
||||
summary += " — “" + truncate(status.Subject, 80) + "”"
|
||||
}
|
||||
|
||||
createdAt := status.LastCommit
|
||||
if createdAt == "" {
|
||||
createdAt = gitStatusTime(status)
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: g.SourceID(),
|
||||
SourcePath: repoPath,
|
||||
Project: name,
|
||||
ContentType: "git-status",
|
||||
Title: name,
|
||||
Summary: summary,
|
||||
CreatedAt: createdAt,
|
||||
Confidence: 0.9,
|
||||
IngesterVersion: g.Version,
|
||||
Provenance: map[string]any{
|
||||
"type": "git_status",
|
||||
"path": repoPath,
|
||||
"branch": status.Branch,
|
||||
"head": status.Head,
|
||||
"dirty": status.Dirty,
|
||||
"ahead": status.Ahead,
|
||||
"behind": status.Behind,
|
||||
"last_commit": status.LastCommit,
|
||||
"subject": status.Subject,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func gitStatusTime(s GitStatus) string {
|
||||
if s.LastCommit != "" {
|
||||
return s.LastCommit
|
||||
}
|
||||
return time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func gitStatus(repoPath string) GitStatus {
|
||||
out, err := gitOut(repoPath, "status", "--porcelain")
|
||||
if err != nil {
|
||||
return GitStatus{}
|
||||
}
|
||||
dirty := 0
|
||||
if out != "" {
|
||||
dirty = len(strings.Split(strings.TrimRight(out, "\n"), "\n"))
|
||||
}
|
||||
|
||||
branch, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
head, _ := gitOut(repoPath, "rev-parse", "--short", "HEAD")
|
||||
lastCommit, _ := gitOut(repoPath, "log", "-1", "--format=%cI")
|
||||
subject, _ := gitOut(repoPath, "log", "-1", "--format=%s")
|
||||
|
||||
ahead, behind := 0, 0
|
||||
// ahead/behind only meaningful if there's an upstream
|
||||
if upOut, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD@{upstream}"); upOut != "" {
|
||||
if rev, err := gitOut(repoPath, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err == nil {
|
||||
fields := strings.Fields(rev)
|
||||
if len(fields) == 2 {
|
||||
var a, b int
|
||||
if _, e1 := fmt.Sscanf(fields[0], "%d", &a); e1 == nil {
|
||||
ahead = a
|
||||
}
|
||||
if _, e2 := fmt.Sscanf(fields[1], "%d", &b); e2 == nil {
|
||||
behind = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return GitStatus{
|
||||
Branch: strings.TrimSpace(branch),
|
||||
Head: strings.TrimSpace(head),
|
||||
Dirty: dirty,
|
||||
Ahead: ahead,
|
||||
Behind: behind,
|
||||
LastCommit: strings.TrimSpace(lastCommit),
|
||||
Subject: strings.TrimSpace(subject),
|
||||
}
|
||||
}
|
||||
|
||||
// sanitizeGitField strips ANSI codes and control bytes from git output
|
||||
// (colour/unicode modes can inject escape sequences).
|
||||
var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||||
|
||||
func sanitizeGitField(s string) string {
|
||||
s = ansiRe.ReplaceAllString(s, "")
|
||||
s = strings.ReplaceAll(s, "\x1b", "")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// gitOut runs git in repoPath and returns trimmed stdout (or "" on error).
|
||||
func gitOut(repoPath string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", append([]string{"-C", repoPath}, args...)...)
|
||||
cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sanitizeGitField(string(out)), nil
|
||||
}
|
||||
|
||||
func (g *GitIngester) Ingest(_ string) (*IngestResult, error) {
|
||||
return nil, fmt.Errorf("use IngestAll() for git")
|
||||
}
|
||||
|
||||
var _ Ingester = (*GitIngester)(nil)
|
||||
|
||||
// SortResults orders results deterministically by project name (helper for CLI output).
|
||||
func SortResults(res []*IngestResult) {
|
||||
sort.Slice(res, func(i, j int) bool { return res[i].Project < res[j].Project })
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
type GiteaIngester struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func NewGiteaIngester() *GiteaIngester {
|
||||
return &GiteaIngester{Version: "gitea-tea/v1"}
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) SourceID() string { return "gitea" }
|
||||
|
||||
func (g *GiteaIngester) IngestAll() ([]*IngestResult, error) {
|
||||
var results []*IngestResult
|
||||
|
||||
repos, err := g.fetchRepos()
|
||||
if err == nil {
|
||||
for _, r := range repos {
|
||||
if result := g.repoToResult(r); result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
issues, err := g.fetchIssues()
|
||||
if err == nil {
|
||||
for _, i := range issues {
|
||||
if result := g.issueToResult(i); result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pulls, err := g.fetchPulls()
|
||||
if err == nil {
|
||||
for _, p := range pulls {
|
||||
if result := g.pullToResult(p); result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type teaRepo struct {
|
||||
Owner string `json:"owner"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
type teaIssue struct {
|
||||
Index string `json:"index"`
|
||||
Title string `json:"title"`
|
||||
State string `json:"state"`
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Labels string `json:"labels"`
|
||||
Author string `json:"author"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
type teaPull struct {
|
||||
Index string `json:"index"`
|
||||
Title string `json:"title"`
|
||||
State string `json:"state"`
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) teaJSON(args ...string) ([][]byte, error) {
|
||||
cmd := exec.Command("tea", append(args, "--output", "json")...)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tea %s: %w", args[0], err)
|
||||
}
|
||||
|
||||
var raw json.RawMessage
|
||||
if err := json.Unmarshal(output, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Handle both array and object responses
|
||||
var arr []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &arr); err == nil {
|
||||
bytes := make([][]byte, len(arr))
|
||||
for i, a := range arr {
|
||||
bytes[i] = a
|
||||
}
|
||||
return bytes, nil
|
||||
}
|
||||
|
||||
// Single object — wrap in array
|
||||
return [][]byte{raw}, nil
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) fetchRepos() ([]teaRepo, error) {
|
||||
items, err := g.teaJSON("repos", "list", "--limit", "100", "--fields", "owner,name,description,updated")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var repos []teaRepo
|
||||
for _, item := range items {
|
||||
var r teaRepo
|
||||
if err := json.Unmarshal(item, &r); err == nil {
|
||||
repos = append(repos, r)
|
||||
}
|
||||
}
|
||||
return repos, nil
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) fetchIssues() ([]teaIssue, error) {
|
||||
items, err := g.teaJSON("issues", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,author,milestone,labels,owner,repo,updated")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var issues []teaIssue
|
||||
for _, item := range items {
|
||||
var i teaIssue
|
||||
if err := json.Unmarshal(item, &i); err == nil {
|
||||
issues = append(issues, i)
|
||||
}
|
||||
}
|
||||
return issues, nil
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) fetchPulls() ([]teaPull, error) {
|
||||
items, err := g.teaJSON("pulls", "list", "--state", "open", "--limit", "100", "--fields", "index,title,state,owner,repo,updated")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pulls []teaPull
|
||||
for _, item := range items {
|
||||
var p teaPull
|
||||
if err := json.Unmarshal(item, &p); err == nil {
|
||||
pulls = append(pulls, p)
|
||||
}
|
||||
}
|
||||
return pulls, nil
|
||||
}
|
||||
|
||||
// signalTime returns the item's last-activity time, falling back to now.
|
||||
func signalTime(updated string) string {
|
||||
if updated != "" {
|
||||
if _, err := time.Parse(time.RFC3339, updated); err == nil {
|
||||
return updated
|
||||
}
|
||||
}
|
||||
return time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) repoToResult(r teaRepo) *IngestResult {
|
||||
fullName := r.Owner + "/" + r.Name
|
||||
fp := Fingerprint([]byte("gitea:repo:" + fullName))
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: g.SourceID(),
|
||||
SourcePath: fullName,
|
||||
Project: r.Name,
|
||||
ContentType: "repo",
|
||||
Title: fullName,
|
||||
Summary: r.Description,
|
||||
Confidence: 0.9,
|
||||
IngesterVersion: g.Version,
|
||||
CreatedAt: signalTime(r.Updated),
|
||||
Provenance: map[string]any{
|
||||
"type": "repository",
|
||||
"owner": r.Owner,
|
||||
"name": r.Name,
|
||||
"description": r.Description,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) issueToResult(i teaIssue) *IngestResult {
|
||||
title := fmt.Sprintf("#%s %s", i.Index, i.Title)
|
||||
fp := Fingerprint([]byte(fmt.Sprintf("gitea:issue:%s/%s:%s", i.Owner, i.Repo, i.Index)))
|
||||
|
||||
summary := fmt.Sprintf("[%s/%s#%s] %s", i.Owner, i.Repo, i.Index, i.Title)
|
||||
if i.Labels != "" {
|
||||
summary += " [" + i.Labels + "]"
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: g.SourceID(),
|
||||
SourcePath: fmt.Sprintf("%s/%s#%s", i.Owner, i.Repo, i.Index),
|
||||
Project: i.Repo,
|
||||
ContentType: "issue",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
Confidence: 0.85,
|
||||
IngesterVersion: g.Version,
|
||||
CreatedAt: signalTime(i.Updated),
|
||||
Provenance: map[string]any{
|
||||
"type": "issue",
|
||||
"owner": i.Owner,
|
||||
"repo": i.Repo,
|
||||
"index": i.Index,
|
||||
"state": i.State,
|
||||
"labels": i.Labels,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) pullToResult(p teaPull) *IngestResult {
|
||||
title := fmt.Sprintf("!%s %s", p.Index, p.Title)
|
||||
fp := Fingerprint([]byte(fmt.Sprintf("gitea:pull:%s/%s:%s", p.Owner, p.Repo, p.Index)))
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: g.SourceID(),
|
||||
SourcePath: fmt.Sprintf("%s/%s!%s", p.Owner, p.Repo, p.Index),
|
||||
Project: p.Repo,
|
||||
ContentType: "pull",
|
||||
Title: title,
|
||||
Summary: fmt.Sprintf("[PR %s/%s] %s", p.Owner, p.Repo, p.Title),
|
||||
Confidence: 0.85,
|
||||
IngesterVersion: g.Version,
|
||||
CreatedAt: signalTime(p.Updated),
|
||||
Provenance: map[string]any{
|
||||
"type": "pull_request",
|
||||
"owner": p.Owner,
|
||||
"repo": p.Repo,
|
||||
"index": p.Index,
|
||||
"state": p.State,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GiteaIngester) Ingest(_ string) (*IngestResult, error) {
|
||||
return nil, fmt.Errorf("use IngestAll() for gitea")
|
||||
}
|
||||
|
||||
var _ Ingester = (*GiteaIngester)(nil)
|
||||
@@ -0,0 +1,57 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type IngestResult struct {
|
||||
Fingerprint string
|
||||
SourceID string
|
||||
SourcePath string
|
||||
Project string
|
||||
ContentType string
|
||||
Title string
|
||||
Summary string
|
||||
CreatedAt string
|
||||
LineStart int
|
||||
LineEnd int
|
||||
Confidence float64
|
||||
IngesterVersion string
|
||||
Provenance map[string]any
|
||||
}
|
||||
|
||||
type Ingester interface {
|
||||
SourceID() string
|
||||
Ingest(path string) (*IngestResult, error)
|
||||
}
|
||||
|
||||
func Fingerprint(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return fmt.Sprintf("%x", h[:16])
|
||||
}
|
||||
|
||||
func FingerprintWithMeta(data []byte, meta map[string]string) string {
|
||||
h := sha256.New()
|
||||
h.Write(data)
|
||||
enc := json.NewEncoder(h)
|
||||
_ = enc.Encode(meta)
|
||||
return fmt.Sprintf("%x", h.Sum(nil)[:16])
|
||||
}
|
||||
|
||||
func ProvenanceJSON(m map[string]any) string {
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return string(runes[:n]) + "..."
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ Ingester = (*LogIngester)(nil)
|
||||
|
||||
type LogIngester struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func NewLogIngester() *LogIngester {
|
||||
return &LogIngester{Version: "log-parser/v1"}
|
||||
}
|
||||
|
||||
func (l *LogIngester) SourceID() string { return "opencode-log" }
|
||||
|
||||
var (
|
||||
sessionIDRE = regexp.MustCompile(`session[=_ ]?([a-zA-Z0-9_-]+)`)
|
||||
projectIDRE = regexp.MustCompile(`project[=_ ]?([a-zA-Z0-9_.-]+)`)
|
||||
modelRE = regexp.MustCompile(`model[=_ ]?([a-zA-Z0-9_.-]+)`)
|
||||
agentRE = regexp.MustCompile(`agent[=_ ]?([a-zA-Z0-9_.-]+)`)
|
||||
errorRE = regexp.MustCompile(`(?i)(error|fail|exception|panic|timeout)`)
|
||||
)
|
||||
|
||||
func (l *LogIngester) Ingest(path string) (*IngestResult, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open log: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var (
|
||||
sessionIDs []string
|
||||
models []string
|
||||
agents []string
|
||||
errors []string
|
||||
totalLines int
|
||||
firstError int
|
||||
)
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
totalLines++
|
||||
|
||||
if m := sessionIDRE.FindStringSubmatch(line); len(m) > 1 {
|
||||
sessionIDs = append(sessionIDs, m[1])
|
||||
}
|
||||
if m := modelRE.FindStringSubmatch(line); len(m) > 1 {
|
||||
models = append(models, m[1])
|
||||
}
|
||||
if m := agentRE.FindStringSubmatch(line); len(m) > 1 {
|
||||
agents = append(agents, m[1])
|
||||
}
|
||||
if errorRE.MatchString(line) {
|
||||
if len(errors) == 0 {
|
||||
firstError = totalLines
|
||||
}
|
||||
errors = append(errors, truncate(line, 120))
|
||||
}
|
||||
}
|
||||
|
||||
base := filepath.Base(path)
|
||||
fp := Fingerprint([]byte(path))
|
||||
title := fmt.Sprintf("Log %s (%d lines)", base, totalLines)
|
||||
createdAt := ""
|
||||
if fi, err := os.Stat(path); err == nil {
|
||||
createdAt = fi.ModTime().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
var summaryParts []string
|
||||
if len(sessionIDs) > 0 {
|
||||
summaryParts = append(summaryParts, fmt.Sprintf("sessions: %s", uniqueJoin(sessionIDs, 5)))
|
||||
}
|
||||
if len(models) > 0 {
|
||||
summaryParts = append(summaryParts, fmt.Sprintf("models: %s", uniqueJoin(models, 3)))
|
||||
}
|
||||
if len(errors) > 0 {
|
||||
summaryParts = append(summaryParts, fmt.Sprintf("%d errors (line %d)", len(errors), firstError))
|
||||
}
|
||||
summary := strings.Join(summaryParts, " | ")
|
||||
if summary == "" {
|
||||
summary = fmt.Sprintf("%d lines", totalLines)
|
||||
}
|
||||
|
||||
confidence := 0.6
|
||||
if len(errors) > 0 {
|
||||
confidence = 0.8
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: l.SourceID(),
|
||||
SourcePath: path,
|
||||
ContentType: ".log",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CreatedAt: createdAt,
|
||||
Confidence: confidence,
|
||||
IngesterVersion: l.Version,
|
||||
LineEnd: totalLines,
|
||||
Provenance: map[string]any{
|
||||
"session_ids": uniqueSlice(sessionIDs),
|
||||
"models": uniqueSlice(models),
|
||||
"agents": uniqueSlice(agents),
|
||||
"error_count": len(errors),
|
||||
"file_size": fmt.Sprintf("%d lines", totalLines),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func uniqueJoin(items []string, max int) string {
|
||||
uniq := uniqueSlice(items)
|
||||
if len(uniq) > max {
|
||||
uniq = uniq[:max]
|
||||
}
|
||||
return strings.Join(uniq, ", ")
|
||||
}
|
||||
|
||||
func uniqueSlice(items []string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var uniq []string
|
||||
for _, s := range items {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
uniq = append(uniq, s)
|
||||
}
|
||||
}
|
||||
return uniq
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const obsidianConfigPath = ".config/obsidian/obsidian.json"
|
||||
|
||||
type ObsidianIngester struct {
|
||||
Version string
|
||||
Vault string // optional override; empty = auto-detect
|
||||
}
|
||||
|
||||
func NewObsidianIngester(vault string) *ObsidianIngester {
|
||||
return &ObsidianIngester{Version: "obsidian-note/v1", Vault: vault}
|
||||
}
|
||||
|
||||
func (o *ObsidianIngester) SourceID() string { return "obsidian" }
|
||||
|
||||
func (o *ObsidianIngester) IngestAll() ([]*IngestResult, error) {
|
||||
vault := o.Vault
|
||||
if vault == "" {
|
||||
var err error
|
||||
vault, err = detectObsidianVault()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no vault path and auto-detect failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !fileExists(vault) {
|
||||
return nil, fmt.Errorf("vault path %s does not exist", vault)
|
||||
}
|
||||
|
||||
var results []*IngestResult
|
||||
err := filepath.Walk(vault, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil // skip errors per-file
|
||||
}
|
||||
if info.IsDir() {
|
||||
dir := filepath.Base(path)
|
||||
if strings.HasPrefix(dir, ".") || dir == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !strings.HasSuffix(path, ".md") {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := o.ingestNote(path, vault)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if result != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return results, err
|
||||
}
|
||||
|
||||
func (o *ObsidianIngester) ingestNote(path, vault string) (*IngestResult, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || len(data) < 10 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
content := string(data)
|
||||
title := extractObsidianTitle(content, path, vault)
|
||||
tags := extractObsidianTags(content)
|
||||
created, modified := extractObsidianTimestamps(content, path)
|
||||
if created == "" {
|
||||
if fi, err := os.Stat(path); err == nil {
|
||||
created = fi.ModTime().UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
if modified == "" && created != "" {
|
||||
modified = created
|
||||
}
|
||||
|
||||
// Fingerprint by content (so edits create updates)
|
||||
fp := FingerprintWithMeta(data, map[string]string{
|
||||
"source": "obsidian",
|
||||
"path": path,
|
||||
})
|
||||
|
||||
relPath, _ := filepath.Rel(vault, path)
|
||||
summary := extractBodyPreview(content, 200)
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: o.SourceID(),
|
||||
SourcePath: relPath,
|
||||
Project: filepath.Base(vault),
|
||||
ContentType: ".md",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CreatedAt: created,
|
||||
LineStart: 0,
|
||||
LineEnd: len(strings.Split(content, "\n")),
|
||||
Confidence: 0.85,
|
||||
IngesterVersion: o.Version,
|
||||
Provenance: map[string]any{
|
||||
"path": relPath,
|
||||
"tags": tags,
|
||||
"modified_at": modified,
|
||||
"note_title": title,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func DetectObsidianVault() (string, error) { return detectObsidianVault() }
|
||||
func detectObsidianVault() (string, error) {
|
||||
home, _ := os.UserHomeDir()
|
||||
cfgPath := filepath.Join(home, obsidianConfigPath)
|
||||
if !fileExists(cfgPath) {
|
||||
// Fallback: check common vault locations
|
||||
common := []string{
|
||||
filepath.Join(home, "notes"),
|
||||
filepath.Join(home, "Documents", "notes"),
|
||||
filepath.Join(home, "Obsidian"),
|
||||
filepath.Join(home, "vault"),
|
||||
}
|
||||
for _, p := range common {
|
||||
if fileExists(filepath.Join(p, ".obsidian")) {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no obsidian config found at %s and no common vault detected", cfgPath)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(cfgPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var cfg struct {
|
||||
Vaults map[string]struct {
|
||||
Path string `json:"path"`
|
||||
} `json:"vaults"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return "", fmt.Errorf("parse obsidian config: %w", err)
|
||||
}
|
||||
|
||||
for _, v := range cfg.Vaults {
|
||||
if v.Path != "" {
|
||||
return v.Path, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no vault path in obsidian config")
|
||||
}
|
||||
|
||||
func extractObsidianTitle(content, path, vault string) string {
|
||||
// Try frontmatter title first
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "title:") {
|
||||
t := strings.TrimSpace(trimmed[6:])
|
||||
t = strings.Trim(t, "\"'")
|
||||
if t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "aliases:") {
|
||||
break
|
||||
}
|
||||
if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: first H1
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "# ") {
|
||||
return strings.TrimSpace(trimmed[2:])
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: filename
|
||||
rel, _ := filepath.Rel(vault, path)
|
||||
return strings.TrimSuffix(rel, ".md")
|
||||
}
|
||||
|
||||
func extractObsidianTags(content string) []string {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "tags:") {
|
||||
raw := strings.TrimSpace(trimmed[5:])
|
||||
raw = strings.Trim(raw, "\"[] ")
|
||||
var tags []string
|
||||
for _, t := range strings.Split(raw, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
t = strings.Trim(t, "\" ")
|
||||
if t != "" {
|
||||
tags = append(tags, t)
|
||||
}
|
||||
}
|
||||
return tags
|
||||
}
|
||||
if trimmed == "---" && strings.Count(content[:len(content)/2], "---") >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractObsidianTimestamps(content, path string) (created, modified string) {
|
||||
// Try frontmatter dates
|
||||
inFrontmatter := false
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "---" {
|
||||
inFrontmatter = !inFrontmatter
|
||||
continue
|
||||
}
|
||||
if !inFrontmatter {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "created:") || strings.HasPrefix(trimmed, "date:") {
|
||||
created = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
|
||||
created = strings.Trim(created, "\"' ")
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "modified:") || strings.HasPrefix(trimmed, "updated:") {
|
||||
modified = strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
|
||||
modified = strings.Trim(modified, "\"' ")
|
||||
}
|
||||
}
|
||||
return created, modified
|
||||
}
|
||||
|
||||
func extractBodyPreview(content string, maxLen int) string {
|
||||
inFrontmatter := false
|
||||
var body strings.Builder
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "---" {
|
||||
if inFrontmatter {
|
||||
inFrontmatter = false
|
||||
continue
|
||||
}
|
||||
inFrontmatter = true
|
||||
continue
|
||||
}
|
||||
if inFrontmatter {
|
||||
continue
|
||||
}
|
||||
if trimmed != "" && body.Len() < maxLen {
|
||||
if body.Len() > 0 {
|
||||
body.WriteString(" ")
|
||||
}
|
||||
body.WriteString(truncate(trimmed, maxLen-body.Len()))
|
||||
}
|
||||
}
|
||||
result := body.String()
|
||||
if len(result) > maxLen {
|
||||
return result[:maxLen] + "..."
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ Ingester = (*ObsidianFileIngester)(nil)
|
||||
|
||||
type ObsidianFileIngester struct {
|
||||
Version string
|
||||
Vault string
|
||||
}
|
||||
|
||||
func NewObsidianFileIngester(vault string) *ObsidianFileIngester {
|
||||
return &ObsidianFileIngester{Version: "obsidian-file/v1", Vault: vault}
|
||||
}
|
||||
|
||||
func (o *ObsidianFileIngester) SourceID() string { return "obsidian" }
|
||||
|
||||
func (o *ObsidianFileIngester) Ingest(path string) (*IngestResult, error) {
|
||||
if !strings.HasSuffix(path, ".md") {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || len(data) < 10 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
content := string(data)
|
||||
title := extractObsidianTitle(content, path, o.Vault)
|
||||
tags := extractObsidianTags(content)
|
||||
created, _ := extractObsidianTimestamps(content, path)
|
||||
if created == "" {
|
||||
if fi, err := os.Stat(path); err == nil {
|
||||
created = fi.ModTime().UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
|
||||
fp := FingerprintWithMeta(data, map[string]string{
|
||||
"source": "obsidian",
|
||||
"path": path,
|
||||
})
|
||||
|
||||
relPath, _ := filepath.Rel(o.Vault, path)
|
||||
summary := extractBodyPreview(content, 200)
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: o.SourceID(),
|
||||
SourcePath: relPath,
|
||||
Project: filepath.Base(o.Vault),
|
||||
ContentType: ".md",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CreatedAt: created,
|
||||
LineEnd: len(strings.Split(content, "\n")),
|
||||
Confidence: 0.85,
|
||||
IngesterVersion: o.Version,
|
||||
Provenance: map[string]any{
|
||||
"path": relPath,
|
||||
"tags": tags,
|
||||
"note_title": title,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type jsonKind int
|
||||
|
||||
const (
|
||||
jsonObject jsonKind = iota
|
||||
jsonArray
|
||||
)
|
||||
|
||||
var _ Ingester = (*SessionDiffIngester)(nil)
|
||||
|
||||
type SessionDiffIngester struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func NewSessionDiffIngester() *SessionDiffIngester {
|
||||
return &SessionDiffIngester{Version: "session-diff/v1"}
|
||||
}
|
||||
|
||||
func (s *SessionDiffIngester) SourceID() string { return "opencode-session" }
|
||||
|
||||
func (s *SessionDiffIngester) Ingest(path string) (*IngestResult, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read session diff: %w", err)
|
||||
}
|
||||
if len(data) < 10 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
raw, kind := parseJSON(data)
|
||||
if raw == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
sessionID := guessString(raw, "session_id", "id", "sessionId")
|
||||
project := guessString(raw, "project", "project_id", "projectPath", "projectId")
|
||||
title := guessString(raw, "title", "name", "label")
|
||||
createdAt := guessString(raw, "created_at", "timestamp", "date", "started_at")
|
||||
if createdAt == "" {
|
||||
if fi, err := os.Stat(path); err == nil {
|
||||
createdAt = fi.ModTime().UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
|
||||
if sessionID == "" {
|
||||
base := filepath.Base(path)
|
||||
sessionID = strings.TrimSuffix(base, filepath.Ext(base))
|
||||
sessionID = strings.TrimPrefix(sessionID, "ses_")
|
||||
}
|
||||
if title == "" {
|
||||
title = fmt.Sprintf("Session %s", sessionID[:min(8, len(sessionID))])
|
||||
}
|
||||
|
||||
summary := extractSummary(raw, kind)
|
||||
fp := FingerprintWithMeta(data, map[string]string{
|
||||
"session_id": sessionID,
|
||||
"source": "session_diff",
|
||||
})
|
||||
|
||||
confidence := 0.7
|
||||
if kind == jsonArray {
|
||||
confidence = 0.4
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: s.SourceID(),
|
||||
SourcePath: path,
|
||||
Project: project,
|
||||
ContentType: ".json",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CreatedAt: createdAt,
|
||||
LineStart: 0,
|
||||
LineEnd: 0,
|
||||
Confidence: confidence,
|
||||
IngesterVersion: s.Version,
|
||||
Provenance: map[string]any{
|
||||
"session_id": sessionID,
|
||||
"kind": kindName(kind),
|
||||
"field_count": len(raw),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseJSON(data []byte) (map[string]any, jsonKind) {
|
||||
var obj map[string]any
|
||||
if err := json.Unmarshal(data, &obj); err == nil {
|
||||
return obj, jsonObject
|
||||
}
|
||||
var arr []any
|
||||
if err := json.Unmarshal(data, &arr); err == nil {
|
||||
m := make(map[string]any)
|
||||
m["_count"] = float64(len(arr))
|
||||
m["_kind"] = "array"
|
||||
if len(arr) > 0 {
|
||||
if first, ok := arr[0].(map[string]any); ok {
|
||||
for k, v := range first {
|
||||
if str, ok := v.(string); ok && len(str) < 200 {
|
||||
m["_sample_"+k] = str
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m, jsonArray
|
||||
}
|
||||
return nil, jsonObject
|
||||
}
|
||||
|
||||
func extractSummary(raw map[string]any, kind jsonKind) string {
|
||||
if msg, ok := raw["message"].(string); ok && len(msg) > 0 {
|
||||
return truncate(msg, 200)
|
||||
}
|
||||
if msgs, ok := raw["messages"].([]any); ok && len(msgs) > 0 {
|
||||
parts := make([]string, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
if mm, ok := m.(map[string]any); ok {
|
||||
if c, ok := mm["content"].(string); ok {
|
||||
parts = append(parts, truncate(c, 100))
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " | ")
|
||||
}
|
||||
if count, ok := raw["_count"].(float64); ok && kind == jsonArray {
|
||||
return fmt.Sprintf("Array with %.0f entries", count)
|
||||
}
|
||||
b, _ := json.Marshal(raw)
|
||||
return truncate(string(b), 200)
|
||||
}
|
||||
|
||||
func guessString(m map[string]any, keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if v, ok := m[k].(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func kindName(k jsonKind) string {
|
||||
if k == jsonArray {
|
||||
return "array"
|
||||
}
|
||||
return "object"
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var _ Ingester = (*SkillsIngester)(nil)
|
||||
|
||||
type SkillsIngester struct {
|
||||
Version string
|
||||
}
|
||||
|
||||
func NewSkillsIngester() *SkillsIngester {
|
||||
return &SkillsIngester{Version: "skills-index/v1"}
|
||||
}
|
||||
|
||||
func (s *SkillsIngester) SourceID() string { return "skills-catalog" }
|
||||
|
||||
func (s *SkillsIngester) Ingest(path string) (*IngestResult, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read skill: %w", err)
|
||||
}
|
||||
if len(data) < 20 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
content := string(data)
|
||||
title := extractSkillName(content, path)
|
||||
summary := extractSkillSummary(content)
|
||||
tags := extractSkillTags(content)
|
||||
projects := extractProjects(tags)
|
||||
createdAt := extractCreatedAt(content)
|
||||
|
||||
fp := FingerprintWithMeta(data, map[string]string{
|
||||
"source": "skills-catalog",
|
||||
"title": title,
|
||||
})
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: s.SourceID(),
|
||||
SourcePath: path,
|
||||
Project: projects,
|
||||
ContentType: ".md",
|
||||
Title: title,
|
||||
Summary: summary,
|
||||
CreatedAt: createdAt,
|
||||
LineStart: 0,
|
||||
LineEnd: len(strings.Split(content, "\n")),
|
||||
Confidence: 0.9,
|
||||
IngesterVersion: s.Version,
|
||||
Provenance: map[string]any{
|
||||
"skill_name": title,
|
||||
"tags": tags,
|
||||
"line_count": len(strings.Split(content, "\n")),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func extractSkillName(content, path string) string {
|
||||
base := filepath.Base(filepath.Dir(path))
|
||||
if base != "" && base != "." && base != "/" {
|
||||
return base
|
||||
}
|
||||
return strings.TrimSuffix(filepath.Base(path), ".md")
|
||||
}
|
||||
|
||||
func extractSkillSummary(content string) string {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "description:") {
|
||||
desc := strings.TrimSpace(trimmed[len("description:"):])
|
||||
if len(desc) > 200 {
|
||||
desc = desc[:200] + "..."
|
||||
}
|
||||
return desc
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "# ") {
|
||||
continue
|
||||
}
|
||||
if trimmed != "" && !strings.HasPrefix(trimmed, "---") {
|
||||
return truncate(trimmed, 200)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractSkillTags(content string) []string {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "tags:") {
|
||||
raw := strings.TrimSpace(trimmed[len("tags:"):])
|
||||
raw = strings.Trim(raw, "\"[]")
|
||||
var tags []string
|
||||
for _, t := range strings.Split(raw, ",") {
|
||||
t = strings.TrimSpace(t)
|
||||
t = strings.Trim(t, "\" ")
|
||||
if t != "" {
|
||||
tags = append(tags, t)
|
||||
}
|
||||
}
|
||||
return tags
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractProjects(tags []string) string {
|
||||
for _, t := range tags {
|
||||
if strings.HasPrefix(t, "scope:project:") {
|
||||
return strings.TrimPrefix(t, "scope:project:")
|
||||
}
|
||||
if strings.HasPrefix(t, "scope:machine:") {
|
||||
return "[machine]"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractCreatedAt(content string) string {
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "created_at:") || strings.HasPrefix(trimmed, "updated_at:") {
|
||||
return strings.TrimSpace(trimmed[strings.Index(trimmed, ":")+1:])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
{{define "head"}}<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Knox{{if .title}} · {{.title}}{{end}}</title>
|
||||
<script src="https://unpkg.com/htmx.org@2.0.4"></script>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<style>body{font-family:system-ui,sans-serif}</style>
|
||||
</head>
|
||||
<body class="bg-gray-950 text-gray-100 min-h-screen">
|
||||
<header class="border-b border-gray-800 p-4 flex items-center justify-between">
|
||||
<h1 class="text-xl font-bold text-emerald-400"><a href="/">Knox</a>{{if .title}} <span class="text-gray-400 text-sm font-normal">/ {{.title}}</span>{{end}}</h1>
|
||||
<nav class="flex gap-4 text-sm">
|
||||
<a href="/" class="text-gray-300 hover:text-white">Dashboard</a>
|
||||
<a href="/threads" class="text-gray-300 hover:text-white">Threads</a>
|
||||
</nav>
|
||||
</header>
|
||||
{{end}}
|
||||
|
||||
{{define "foot"}}</body>
|
||||
</html>
|
||||
{{end}}
|
||||
|
||||
{{define "not_found"}}
|
||||
{{template "head" .}}
|
||||
<main class="max-w-4xl mx-auto p-6 text-center">
|
||||
<div class="text-gray-500 text-lg">Page not found</div>
|
||||
<a href="/" class="mt-6 inline-block text-emerald-400 hover:text-emerald-300 text-sm">Back to Dashboard</a>
|
||||
</main>
|
||||
{{template "foot" .}}
|
||||
{{end}}
|
||||
|
||||
{{define "dashboard"}}
|
||||
{{template "head" .}}
|
||||
<main class="max-w-6xl mx-auto p-6 space-y-6">
|
||||
{{if .golden}}
|
||||
<div class="bg-emerald-900/30 border border-emerald-800 rounded-lg p-4">
|
||||
<div class="text-xs text-emerald-400 uppercase tracking-wider">Golden Thread</div>
|
||||
<div class="text-lg font-semibold mt-1">{{.golden.Title}}</div>
|
||||
<div class="text-sm text-gray-400">{{.golden.Motivation}}</div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<div class="flex gap-4">
|
||||
<input type="text" name="q" placeholder="Search knowledge index..."
|
||||
class="flex-1 bg-gray-900 border border-gray-700 rounded px-4 py-2 text-sm focus:outline-none focus:border-emerald-500"
|
||||
hx-get="/search" hx-trigger="keyup changed delay:300ms" hx-target="#results" hx-indicator="#search-indicator">
|
||||
<button class="bg-emerald-700 hover:bg-emerald-600 px-4 py-2 rounded text-sm font-medium"
|
||||
hx-get="/search" hx-include="[name=q]" hx-target="#results" hx-indicator="#search-indicator">Search</button>
|
||||
</div>
|
||||
<div id="results" class="text-sm text-gray-500"></div>
|
||||
<div id="search-indicator" class="htmx-indicator text-xs text-emerald-400">Searching...</div>
|
||||
|
||||
<div hx-get="/stats" hx-trigger="load, every 30s" hx-swap="innerHTML">
|
||||
<div class="text-gray-500">Loading stats...</div>
|
||||
</div>
|
||||
|
||||
<section>
|
||||
<h2 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Issues</h2>
|
||||
{{if .gitea}}
|
||||
<div class="space-y-1 mb-4">
|
||||
{{range .gitea}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<span class="text-emerald-300">{{trunc .Title 80}}</span>
|
||||
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Notes</h3>
|
||||
{{if .obsidian}}
|
||||
<div class="space-y-1">
|
||||
{{range .obsidian}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<span>{{trunc .Title 50}}</span>
|
||||
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="text-gray-500 text-xs">No notes indexed.</div>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Sessions</h3>
|
||||
{{if .session}}
|
||||
<div class="space-y-1">
|
||||
{{range .session}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<span>{{trunc .Title 50}}</span>
|
||||
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="text-gray-500 text-xs">No sessions tracked.</div>
|
||||
{{end}}
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Skills</h3>
|
||||
{{if .skills}}
|
||||
<div class="space-y-1">
|
||||
{{range .skills}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<span>{{trunc .Title 50}}</span>
|
||||
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="text-gray-500 text-xs">No skills indexed.</div>
|
||||
{{end}}
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-500 uppercase tracking-wider mb-2">Feed</h3>
|
||||
<div class="space-y-1">
|
||||
{{range .recent}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<span class="font-mono text-xs text-gray-500">{{shortFP .Fingerprint}}</span>
|
||||
<span class="ml-2">{{trunc .Title 60}}</span>
|
||||
<span class="ml-2 text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.SourceID}}</span>
|
||||
<span class="ml-2 text-xs text-gray-600">{{displayTime .}}</span>
|
||||
</a>
|
||||
{{else}}
|
||||
<div class="text-gray-500 text-sm">No entries yet.</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
{{template "foot" .}}
|
||||
{{end}}
|
||||
|
||||
{{define "search_results"}}
|
||||
{{if .results}}
|
||||
<div class="space-y-1">
|
||||
<div class="text-xs text-gray-500 mb-2">{{len .results}} results for "{{.query}}"</div>
|
||||
{{range .results}}
|
||||
<a href="/entries/{{.Fingerprint}}" class="block bg-gray-900 hover:bg-gray-800 rounded px-3 py-2 text-sm">
|
||||
<div>
|
||||
<span class="font-mono text-xs text-gray-500">{{shortFP .Fingerprint}}</span>
|
||||
<span class="ml-2 font-medium">{{trunc .Title 70}}</span>
|
||||
<span class="ml-2 text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.SourceID}}</span>
|
||||
</div>
|
||||
{{if .Summary}}<div class="text-gray-500 text-xs mt-0.5 ml-12">{{trunc .Summary 120}}</div>{{end}}
|
||||
</a>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<p class="text-gray-500 text-sm">No results for "{{.query}}".</p>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{define "stats"}}
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4">
|
||||
{{range .stats}}
|
||||
<div class="bg-gray-900 rounded-lg p-3 border border-gray-800">
|
||||
<div class="text-xs text-gray-500 uppercase tracking-wider">{{.Label}}</div>
|
||||
<div class="text-xl font-semibold mt-1">{{.Value}}</div>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{define "thread_list"}}
|
||||
{{template "head" .}}
|
||||
<main class="max-w-4xl mx-auto p-6 space-y-4">
|
||||
<div class="flex gap-2 mb-4">
|
||||
<a href="/threads" class="px-3 py-1 text-sm rounded {{if eq .status ""}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">All</a>
|
||||
<a href="/threads?status=active" class="px-3 py-1 text-sm rounded {{if eq .status "active"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Active</a>
|
||||
<a href="/threads?status=stalled" class="px-3 py-1 text-sm rounded {{if eq .status "stalled"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Stalled</a>
|
||||
<a href="/threads?status=resolved" class="px-3 py-1 text-sm rounded {{if eq .status "resolved"}}bg-emerald-700{{else}}bg-gray-800 hover:bg-gray-700{{end}}">Resolved</a>
|
||||
</div>
|
||||
{{range .threads}}
|
||||
<a href="/threads/{{.ID}}" class="block bg-gray-900 rounded-lg p-4 border border-gray-800 hover:border-gray-600">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="text-lg font-semibold">{{.Title}}</span>
|
||||
<span class="text-xs px-1.5 py-0.5 rounded {{if eq .Status "active"}}bg-emerald-900 text-emerald-300{{else if eq .Status "resolved"}}bg-blue-900 text-blue-300{{else}}bg-gray-800 text-gray-400{{end}}">{{.Status}}</span>
|
||||
<span class="text-xs text-gray-500">{{.Priority}}</span>
|
||||
</div>
|
||||
{{if .Motivation}}<div class="text-sm text-gray-400 mt-1">{{.Motivation}}</div>{{end}}
|
||||
<div class="text-xs text-gray-600 mt-1">{{.EntryCount}} observations · {{humanTime .UpdatedAt}}</div>
|
||||
</a>
|
||||
{{else}}
|
||||
<div class="text-gray-500 text-sm">No threads found.</div>
|
||||
{{end}}
|
||||
</main>
|
||||
{{template "foot" .}}
|
||||
{{end}}
|
||||
|
||||
{{define "thread_detail"}}
|
||||
{{template "head" .}}
|
||||
<main class="max-w-4xl mx-auto p-6 space-y-6">
|
||||
<div class="bg-gray-900 rounded-lg p-4 border border-gray-800">
|
||||
<div class="flex items-center gap-2">
|
||||
<h2 class="text-xl font-bold">{{.thread.Title}}</h2>
|
||||
{{if .isGolden}}<span class="text-xs bg-emerald-900 text-emerald-300 px-2 py-0.5 rounded">GOLDEN</span>{{end}}
|
||||
</div>
|
||||
<div class="flex gap-4 mt-2 text-sm text-gray-400">
|
||||
<span>{{.thread.Status}}</span>
|
||||
<span>{{.thread.Priority}}</span>
|
||||
<span>Created {{humanTime .thread.CreatedAt}}</span>
|
||||
</div>
|
||||
{{if .thread.Motivation}}<div class="mt-3 text-gray-300"><strong>Motivation:</strong> {{.thread.Motivation}}</div>{{end}}
|
||||
{{if ne .thread.Provenance "{}"}}<div class="mt-1 text-xs text-gray-500"><strong>Provenance:</strong> {{.thread.Provenance}}</div>{{end}}
|
||||
</div>
|
||||
|
||||
{{if .provenance}}
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Provenance Graph</h3>
|
||||
<div class="bg-gray-900 rounded-lg p-3 border border-gray-800 text-sm font-mono">
|
||||
{{range .provenance}}
|
||||
<div class="text-gray-400">└─ <a href="{{fpURL .Fingerprint}}" class="text-emerald-400 hover:text-emerald-300">{{shortFP .Fingerprint}}</a> <span class="text-gray-600">[{{.Relation}}]</span></div>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{if .obs}}
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Observations ({{len .obs}})</h3>
|
||||
<div class="space-y-1">
|
||||
{{range .obs}}
|
||||
<div class="bg-gray-900 rounded px-3 py-2 text-sm flex items-center gap-3">
|
||||
<span class="font-mono text-xs text-gray-500">#{{.ID}}</span>
|
||||
<span class="flex-1">{{trunc .Title 50}}</span>
|
||||
<span class="text-xs text-gray-500">{{.SourceID}}</span>
|
||||
{{if gt .Count 1}}<span class="text-xs text-amber-500" title="seen {{.Count}} times">×{{.Count}}</span>{{end}}
|
||||
<span class="text-xs text-gray-600">{{humanTime .CollectedAt}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{if .notes}}
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Notes</h3>
|
||||
<div class="space-y-2">
|
||||
{{range .notes}}
|
||||
<div class="bg-gray-900 rounded p-3 border border-gray-800">
|
||||
<div class="text-xs text-gray-500">{{humanTime .CreatedAt}}</div>
|
||||
<div class="text-sm mt-1">{{.Note}}</div>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
</main>
|
||||
{{template "foot" .}}
|
||||
{{end}}
|
||||
|
||||
{{define "entry_detail"}}
|
||||
{{template "head" .}}
|
||||
<main class="max-w-4xl mx-auto p-6 space-y-4">
|
||||
<div class="bg-gray-900 rounded-lg p-4 border border-gray-800">
|
||||
<div class="grid grid-cols-2 gap-4 text-sm">
|
||||
<div><span class="text-gray-500">Fingerprint</span><br><span class="font-mono text-xs">{{.entry.Fingerprint}}</span></div>
|
||||
<div><span class="text-gray-500">Source</span><br>{{.entry.SourceID}}</div>
|
||||
<div><span class="text-gray-500">Title</span><br><span class="font-medium">{{.entry.Title}}</span></div>
|
||||
<div><span class="text-gray-500">Project</span><br>{{.entry.Project}}</div>
|
||||
<div><span class="text-gray-500">Path</span><br><span class="font-mono text-xs">{{.entry.SourcePath}}</span></div>
|
||||
<div><span class="text-gray-500">Type</span><br>{{.entry.ContentType}}</div>
|
||||
<div><span class="text-gray-500">Signal Time</span><br>{{if .entry.CreatedAt}}{{humanTime .entry.CreatedAt}}{{else}}<span class="text-gray-500">—</span>{{end}}</div>
|
||||
<div><span class="text-gray-500">Ingested</span><br>{{humanTime .entry.LastSeen}}</div>
|
||||
<div><span class="text-gray-500">Ref Count</span><br>{{.entry.RefCount}}</div>
|
||||
<div><span class="text-gray-500">Confidence</span><br>{{printf "%.1f" .entry.LastConfidence}}</div>
|
||||
</div>
|
||||
{{if .entry.Summary}}<div class="mt-4 p-3 bg-gray-800 rounded text-sm">{{.entry.Summary}}</div>{{end}}
|
||||
</div>
|
||||
|
||||
{{if .obs}}
|
||||
<section>
|
||||
<h3 class="text-sm font-semibold text-gray-400 uppercase tracking-wider mb-2">Observation History ({{len .obs}})</h3>
|
||||
<div class="space-y-1">
|
||||
{{range .obs}}
|
||||
<div class="bg-gray-900 rounded px-3 py-2 text-sm flex items-center gap-3">
|
||||
<span class="font-mono text-xs text-gray-500">#{{.ID}}</span>
|
||||
<span class="text-xs text-gray-500">{{humanTime .CollectedAt}}</span>
|
||||
{{if gt .Count 1}}<span class="text-xs text-amber-500" title="seen {{.Count}} times, unchanged">×{{.Count}}</span>{{end}}
|
||||
<span class="text-xs px-1.5 py-0.5 rounded bg-gray-800 text-gray-400">{{.Trigger}}</span>
|
||||
<span class="text-xs text-gray-500">{{.IngesterVersion}}</span>
|
||||
<span class="text-xs text-gray-500">conf:{{printf "%.1f" .Confidence}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</section>
|
||||
{{end}}
|
||||
</main>
|
||||
{{template "foot" .}}
|
||||
{{end}}
|
||||
@@ -0,0 +1,283 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"log"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
kdb *db.KnoxDB
|
||||
mux *http.ServeMux
|
||||
tmpl *template.Template
|
||||
}
|
||||
|
||||
func New(kdb *db.KnoxDB) *Server {
|
||||
s := &Server{kdb: kdb, mux: http.NewServeMux()}
|
||||
s.tmpl = template.Must(template.New("knox").Funcs(template.FuncMap{
|
||||
"shortFP": shortFP,
|
||||
"trunc": truncStr,
|
||||
"fpURL": fpURL,
|
||||
"humanTime": humanTime,
|
||||
"displayTime": displayTime,
|
||||
}).Parse(layout))
|
||||
|
||||
s.mux.HandleFunc("GET /", s.dashboard)
|
||||
s.mux.HandleFunc("GET /search", s.search)
|
||||
s.mux.HandleFunc("GET /threads", s.threadList)
|
||||
s.mux.HandleFunc("GET /threads/{id}", s.threadDetail)
|
||||
s.mux.HandleFunc("GET /entries/{fp}", s.entryDetail)
|
||||
s.mux.HandleFunc("GET /stats", s.stats)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) Serve(addr string) error {
|
||||
log.Printf("[knox] web UI at http://%s", addr)
|
||||
return http.ListenAndServe(addr, s.mux)
|
||||
}
|
||||
|
||||
// ─── Dashboard ───────────────────────────────────────────────
|
||||
|
||||
func (s *Server) dashboard(w http.ResponseWriter, r *http.Request) {
|
||||
gitea, _ := s.kdb.RecentEntriesBySource("gitea", 3)
|
||||
obsidian, _ := s.kdb.RecentEntriesBySource("obsidian", 3)
|
||||
skills, _ := s.kdb.RecentEntriesBySource("skills-catalog", 2)
|
||||
session, _ := s.kdb.RecentEntriesBySource("opencode-session", 3)
|
||||
recent, _ := s.kdb.RecentEntries(8)
|
||||
|
||||
goldenID, err := s.kdb.GoldenThreadID()
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load golden thread", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var golden *db.Thread
|
||||
if goldenID > 0 {
|
||||
golden, _ = s.kdb.GetThread(goldenID)
|
||||
}
|
||||
s.render(w, "dashboard", map[string]any{
|
||||
"gitea": gitea,
|
||||
"obsidian": obsidian,
|
||||
"skills": skills,
|
||||
"session": session,
|
||||
"recent": recent,
|
||||
"golden": golden,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Search ──────────────────────────────────────────────────
|
||||
|
||||
func (s *Server) search(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query().Get("q")
|
||||
if q == "" {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write([]byte("<p class='text-gray-400 text-sm'>Enter a query above.</p>"))
|
||||
return
|
||||
}
|
||||
results, err := s.kdb.Search(q, 30)
|
||||
if err != nil {
|
||||
http.Error(w, "search failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.render(w, "search_results", map[string]any{
|
||||
"query": q,
|
||||
"results": results,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Thread List ─────────────────────────────────────────────
|
||||
|
||||
func (s *Server) threadList(w http.ResponseWriter, r *http.Request) {
|
||||
status := r.URL.Query().Get("status")
|
||||
threads, err := s.kdb.ListThreads(status)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load threads", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.render(w, "thread_list", map[string]any{
|
||||
"title": "Threads",
|
||||
"threads": threads,
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Thread Detail ───────────────────────────────────────────
|
||||
|
||||
func (s *Server) threadDetail(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid thread id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
t, err := s.kdb.GetThread(id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load thread", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if t == nil {
|
||||
s.render(w, "not_found", map[string]any{"title": "Thread Not Found"}, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
obs, err := s.kdb.ThreadObservations(id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
notes, err := s.kdb.ThreadNotes(id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load notes", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
provenance, err := s.kdb.ThreadProvenance(id)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load provenance", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
goldenID, _ := s.kdb.GoldenThreadID()
|
||||
|
||||
s.render(w, "thread_detail", map[string]any{
|
||||
"title": fmt.Sprintf("Thread #%d", id),
|
||||
"thread": t,
|
||||
"obs": obs,
|
||||
"notes": notes,
|
||||
"provenance": provenance,
|
||||
"isGolden": goldenID == id,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Entry Detail ────────────────────────────────────────────
|
||||
|
||||
func (s *Server) entryDetail(w http.ResponseWriter, r *http.Request) {
|
||||
fp := r.PathValue("fp")
|
||||
entry, err := s.kdb.FindEntry(fp)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load entry", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if entry == nil {
|
||||
// Try fingerprint prefix match (the UI displays 8-char prefixes)
|
||||
entries, err := s.kdb.FindEntryByPrefix(fp, 2)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load entry", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if len(entries) == 1 {
|
||||
entry = &entries[0]
|
||||
} else {
|
||||
s.render(w, "not_found", map[string]any{"title": "Entry Not Found"}, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
obs, err := s.kdb.ObservationsByFingerprint(entry.Fingerprint, 20)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load observations", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.render(w, "entry_detail", map[string]any{
|
||||
"title": "Entry " + shortFP(entry.Fingerprint),
|
||||
"entry": entry,
|
||||
"obs": obs,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Stats Fragment ──────────────────────────────────────────
|
||||
|
||||
type statItem struct {
|
||||
Label string
|
||||
Value any
|
||||
}
|
||||
|
||||
func statItems(stats map[string]any) []statItem {
|
||||
keys := make([]string, 0, len(stats))
|
||||
for k := range stats {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
items := make([]statItem, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
items = append(items, statItem{
|
||||
Label: strings.ReplaceAll(k, "_", " "),
|
||||
Value: stats[k],
|
||||
})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (s *Server) stats(w http.ResponseWriter, r *http.Request) {
|
||||
stats, err := s.kdb.Stats()
|
||||
if err != nil {
|
||||
http.Error(w, "failed to load stats", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.render(w, "stats", map[string]any{"stats": statItems(stats)})
|
||||
}
|
||||
|
||||
// ─── Render ──────────────────────────────────────────────────
|
||||
|
||||
func (s *Server) render(w http.ResponseWriter, name string, data map[string]any, status ...int) {
|
||||
code := http.StatusOK
|
||||
if len(status) > 0 {
|
||||
code = status[0]
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := s.tmpl.ExecuteTemplate(&buf, name, data); err != nil {
|
||||
log.Printf("[knox] template error: %v", err)
|
||||
http.Error(w, "template error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(code)
|
||||
buf.WriteTo(w)
|
||||
}
|
||||
|
||||
func shortFP(fp string) string {
|
||||
if len(fp) > 8 {
|
||||
return fp[:8]
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
// fpURL routes synthetic thread fingerprints to thread pages.
|
||||
func fpURL(fp string) string {
|
||||
if id, ok := strings.CutPrefix(fp, "thread:"); ok {
|
||||
return "/threads/" + id
|
||||
}
|
||||
return "/entries/" + fp
|
||||
}
|
||||
|
||||
func truncStr(s string, n int) string {
|
||||
runes := []rune(s)
|
||||
if len(runes) <= n {
|
||||
return s
|
||||
}
|
||||
return string(runes[:n]) + "..."
|
||||
}
|
||||
|
||||
func displayTime(e db.Entry) string {
|
||||
if e.CreatedAt != "" {
|
||||
return humanTime(e.CreatedAt)
|
||||
}
|
||||
return humanTime(e.LastSeen)
|
||||
}
|
||||
|
||||
func humanTime(s string) string {
|
||||
formats := []string{time.RFC3339, "2006-01-02 15:04:05"}
|
||||
for _, f := range formats {
|
||||
t, err := time.Parse(f, s)
|
||||
if err == nil {
|
||||
return t.Local().Format("Jan 2, 2006 · 15:04")
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
//go:embed layout.html
|
||||
var layout string
|
||||
@@ -0,0 +1,311 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/index"
|
||||
)
|
||||
|
||||
// AutoThreader promotes high-intent topic clusters into knowledge threads.
|
||||
//
|
||||
// Detection is entirely heuristic (no LLM): a TF-IDF topic cluster crosses the
|
||||
// "intent bar" when it shows sustained, recently-observed activity from a
|
||||
// non-passive work source (git/gitea/session/obsidian) plus enough distinct
|
||||
// sources overall. Motivation and title are synthesized from the cluster's
|
||||
// strongest members — they can be refined later, but the grouping and linkage
|
||||
// are deterministic and cheap enough for the watch daemon's periodic tick.
|
||||
type AutoThreader struct {
|
||||
DB *db.KnoxDB
|
||||
MinSources int // distinct sources required (incl. one work source)
|
||||
MinEntries int // minimum cluster size
|
||||
Recency time.Duration // observations must be seen within this window
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// workSources are signal sources that imply active intent (vs passive browsing
|
||||
// or an always-present skills catalog).
|
||||
var workSources = map[string]bool{
|
||||
"git": true,
|
||||
"gitea": true,
|
||||
"opencode-session": true,
|
||||
"obsidian": true,
|
||||
"filesystem": true,
|
||||
}
|
||||
|
||||
// NewAutoThreader reads the tuning knobs from the environment. All have
|
||||
// defaults; set KNOX_THREAD_DISABLE=1 to turn auto-threading off.
|
||||
func NewAutoThreader(kdb *db.KnoxDB) *AutoThreader {
|
||||
t := &AutoThreader{
|
||||
DB: kdb,
|
||||
MinSources: envInt("KNOX_THREAD_MIN_SOURCES", 2),
|
||||
MinEntries: envInt("KNOX_THREAD_MIN_ENTRIES", 3),
|
||||
Recency: time.Duration(envInt("KNOX_THREAD_RECENCY_HOURS", 24*7)) * time.Hour,
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
if v, err := strconv.Atoi(os.Getenv(key)); err == nil {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// AutoThread clusters recent entries and creates/extends threads for clusters
|
||||
// crossing the intent bar. Returns the number of threads created and the total
|
||||
// observations linked.
|
||||
func (t *AutoThreader) AutoThread() (created, linked int, err error) {
|
||||
entries, err := t.DB.RecentEntries(2000)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
tfidf := index.BuildTFIDF(entries)
|
||||
clusters := tfidf.Cluster(2, 30)
|
||||
now := time.Now().UTC()
|
||||
|
||||
for _, c := range clusters {
|
||||
fps := fingerprints(c.Entries)
|
||||
if len(fps) == 0 || !t.crossesBar(c, fps, now) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Fold into an existing active thread if one already covers these terms.
|
||||
if existing := t.findExisting(c); existing > 0 {
|
||||
if t.DryRun {
|
||||
fmt.Printf("[threader] (dry-run) would link %d obs into existing thread #%d: %s\n", len(fps), existing, c.Name)
|
||||
continue
|
||||
}
|
||||
n, err := t.DB.AutoLinkThreadObservations(existing, fps)
|
||||
if err != nil {
|
||||
log.Printf("[knox] threader link err: %v", err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("[knox] auto-thread #%d extended: %s (+%d obs)", existing, c.Name, n)
|
||||
}
|
||||
linked += n
|
||||
continue
|
||||
}
|
||||
|
||||
title := t.title(c)
|
||||
motivation := t.motivation(c, fps)
|
||||
tags := t.tags(c)
|
||||
priority := t.priority(c)
|
||||
prov := map[string]any{
|
||||
"trigger": "auto_thread",
|
||||
"cluster": c.Name,
|
||||
"score": c.Score,
|
||||
"sources": sourceList(c.Entries),
|
||||
}
|
||||
provJSON, _ := json.Marshal(prov)
|
||||
|
||||
if t.DryRun {
|
||||
fmt.Printf("[threader] (dry-run) would create thread %q [%s]\n", title, priority)
|
||||
fmt.Printf(" motivation: %s\n", motivation)
|
||||
fmt.Printf(" tags: %s\n", tags)
|
||||
fmt.Printf(" %d obs | sources: %v\n", len(fps), prov["sources"])
|
||||
continue
|
||||
}
|
||||
|
||||
id, err := t.DB.CreateThread(title, motivation, priority, tags, string(provJSON))
|
||||
if err != nil {
|
||||
log.Printf("[knox] threader create err: %v", err)
|
||||
continue
|
||||
}
|
||||
if n, err := t.DB.AutoLinkThreadObservations(id, fps); err == nil {
|
||||
linked += n
|
||||
}
|
||||
created++
|
||||
log.Printf("[knox] auto-thread #%d: %s (%d obs, %s)", id, title, len(fps), priority)
|
||||
}
|
||||
return created, linked, nil
|
||||
}
|
||||
|
||||
// crossesBar decides whether a cluster reflects real intent.
|
||||
func (t *AutoThreader) crossesBar(c index.TopicCluster, fps []string, now time.Time) bool {
|
||||
if len(c.Entries) < t.MinEntries {
|
||||
return false
|
||||
}
|
||||
|
||||
bySource := make(map[string]bool)
|
||||
hasWork := false
|
||||
recent := false
|
||||
for _, e := range c.Entries {
|
||||
if e.SourceID != "" {
|
||||
bySource[e.SourceID] = true
|
||||
if workSources[e.SourceID] {
|
||||
hasWork = true
|
||||
}
|
||||
}
|
||||
if e.LastSeen != "" {
|
||||
if ts, err := time.Parse(time.RFC3339, e.LastSeen); err == nil && now.Sub(ts) <= t.Recency {
|
||||
recent = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(bySource) < t.MinSources {
|
||||
return false
|
||||
}
|
||||
if !hasWork {
|
||||
return false
|
||||
}
|
||||
return recent
|
||||
}
|
||||
|
||||
// findExisting looks for an already-active thread that plausibly covers this
|
||||
// cluster, by matching a discriminative cluster keyword against thread titles.
|
||||
func (t *AutoThreader) findExisting(c index.TopicCluster) int64 {
|
||||
for _, kw := range c.Keywords {
|
||||
id, err := t.DB.ActiveThreadByKeyword(kw)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
if id == 0 {
|
||||
continue
|
||||
}
|
||||
th, err := t.DB.GetThread(id)
|
||||
if err != nil || th == nil {
|
||||
continue
|
||||
}
|
||||
titleTokens := toSet(strings.ToLower(th.Title))
|
||||
for _, kw2 := range c.Keywords {
|
||||
if titleTokens[kw2] {
|
||||
return id
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (t *AutoThreader) title(c index.TopicCluster) string {
|
||||
return c.Name
|
||||
}
|
||||
|
||||
func (t *AutoThreader) tags(c index.TopicCluster) string {
|
||||
var keep []string
|
||||
for _, kw := range c.Keywords {
|
||||
if len(keep) >= 5 {
|
||||
break
|
||||
}
|
||||
keep = append(keep, kw)
|
||||
}
|
||||
return strings.Join(keep, " ")
|
||||
}
|
||||
|
||||
func (t *AutoThreader) priority(c index.TopicCluster) string {
|
||||
srcs := len(sourceList(c.Entries))
|
||||
if srcs >= 3 {
|
||||
return "high"
|
||||
}
|
||||
return "medium"
|
||||
}
|
||||
|
||||
// motivation synthesizes "why" from the cluster's strongest members: project
|
||||
// context, source spread, and a couple of representative titles/summaries.
|
||||
func (t *AutoThreader) motivation(c index.TopicCluster, fps []string) string {
|
||||
sources := sourceList(c.Entries)
|
||||
|
||||
proj := dominantProject(c.Entries)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Auto-detected cluster of %d observations across %d sources (%s).", len(c.Entries), len(sources), strings.Join(sources, ", "))
|
||||
if proj != "" {
|
||||
fmt.Fprintf(&b, " Dominant project: %s.", proj)
|
||||
}
|
||||
fmt.Fprintf(&b, " Keywords: %s.", c.Name)
|
||||
fmt.Fprintf(&b, " Representative observations:")
|
||||
|
||||
shown := 0
|
||||
seen := make(map[string]bool)
|
||||
for _, e := range c.Entries {
|
||||
if shown >= 3 {
|
||||
break
|
||||
}
|
||||
txt := strings.TrimSpace(e.Title)
|
||||
if txt == "" {
|
||||
txt = strings.TrimSpace(e.Summary)
|
||||
}
|
||||
if txt == "" || seen[txt] {
|
||||
continue
|
||||
}
|
||||
seen[txt] = true
|
||||
fmt.Fprintf(&b, " • %s", truncate(txt, 90))
|
||||
shown++
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// --- small helpers ---------------------------------------------------------
|
||||
|
||||
func fingerprints(entries []db.Entry) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.Fingerprint == "" || seen[e.Fingerprint] {
|
||||
continue
|
||||
}
|
||||
seen[e.Fingerprint] = true
|
||||
out = append(out, e.Fingerprint)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sourceList(entries []db.Entry) []string {
|
||||
seen := make(map[string]bool)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.SourceID == "" || seen[e.SourceID] {
|
||||
continue
|
||||
}
|
||||
seen[e.SourceID] = true
|
||||
out = append(out, e.SourceID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dominantProject(entries []db.Entry) string {
|
||||
counts := make(map[string]int)
|
||||
for _, e := range entries {
|
||||
if e.Project != "" {
|
||||
counts[e.Project]++
|
||||
}
|
||||
}
|
||||
best, n := "", 0
|
||||
for p, c := range counts {
|
||||
if c > n {
|
||||
best, n = p, c
|
||||
}
|
||||
}
|
||||
if n*2 >= len(entries) {
|
||||
return best
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func toSet(s string) map[string]bool {
|
||||
set := make(map[string]bool)
|
||||
for _, w := range strings.Fields(s) {
|
||||
set[strings.Trim(strings.ToLower(w), ",.:;()/[]{}")] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/david/knox/internal/db"
|
||||
"github.com/david/knox/internal/ingest"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
const browserInterval = 5 * time.Minute
|
||||
|
||||
type Watcher struct {
|
||||
knoxDB *db.KnoxDB
|
||||
dirs []string
|
||||
vault string
|
||||
debounce time.Duration
|
||||
fileIngesters []ingest.Ingester
|
||||
}
|
||||
|
||||
func New(kdb *db.KnoxDB, dirs []string) *Watcher {
|
||||
// Detect Obsidian vault
|
||||
vault, _ := ingest.DetectObsidianVault()
|
||||
|
||||
return &Watcher{
|
||||
knoxDB: kdb,
|
||||
dirs: dirs,
|
||||
vault: vault,
|
||||
debounce: 2 * time.Second,
|
||||
fileIngesters: []ingest.Ingester{
|
||||
ingest.NewSessionDiffIngester(),
|
||||
ingest.NewLogIngester(),
|
||||
ingest.NewSkillsIngester(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) Start() error {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer watcher.Close()
|
||||
|
||||
w.seed(watcher)
|
||||
log.Printf("[knox] watching %d directories", len(w.dirs))
|
||||
|
||||
if w.vault != "" {
|
||||
log.Printf("[knox] obsidian vault: %s", w.vault)
|
||||
}
|
||||
|
||||
debounceMap := make(map[string]time.Time)
|
||||
browserTicker := time.NewTicker(browserInterval)
|
||||
giteaTicker := time.NewTicker(10 * time.Minute)
|
||||
gitTicker := time.NewTicker(10 * time.Minute)
|
||||
threadTicker := time.NewTicker(10 * time.Minute)
|
||||
browserRunning := false
|
||||
giteaRunning := false
|
||||
gitRunning := false
|
||||
threadRunning := false
|
||||
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-watcher.Events:
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if !w.isRelevantEvent(event) {
|
||||
continue
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if last, ok := debounceMap[event.Name]; ok && now.Sub(last) < w.debounce {
|
||||
continue
|
||||
}
|
||||
debounceMap[event.Name] = now
|
||||
|
||||
trigger := eventOpName(event.Op)
|
||||
log.Printf("[knox] %s %s", trigger, filepath.Base(event.Name))
|
||||
w.ingestFile(event.Name, trigger)
|
||||
|
||||
case <-browserTicker.C:
|
||||
if browserRunning {
|
||||
continue
|
||||
}
|
||||
browserRunning = true
|
||||
go func() {
|
||||
defer func() { browserRunning = false }()
|
||||
w.ingestBrowserHistory()
|
||||
}()
|
||||
case <-giteaTicker.C:
|
||||
if giteaRunning {
|
||||
continue
|
||||
}
|
||||
giteaRunning = true
|
||||
go func() {
|
||||
defer func() { giteaRunning = false }()
|
||||
w.ingestGitea()
|
||||
}()
|
||||
case <-gitTicker.C:
|
||||
if gitRunning {
|
||||
continue
|
||||
}
|
||||
gitRunning = true
|
||||
go func() {
|
||||
defer func() { gitRunning = false }()
|
||||
w.ingestGit()
|
||||
}()
|
||||
case <-threadTicker.C:
|
||||
if threadRunning {
|
||||
continue
|
||||
}
|
||||
threadRunning = true
|
||||
go func() {
|
||||
defer func() { threadRunning = false }()
|
||||
w.autoThread()
|
||||
}()
|
||||
|
||||
case err, ok := <-watcher.Errors:
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
log.Printf("[knox] watch error: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) seed(watcher *fsnotify.Watcher) {
|
||||
for _, dir := range w.dirs {
|
||||
abs, _ := filepath.Abs(dir)
|
||||
if err := watcher.Add(abs); err != nil {
|
||||
log.Printf("[knox] cannot watch %s: %v", abs, err)
|
||||
continue
|
||||
}
|
||||
log.Printf("[knox] watching %s", abs)
|
||||
}
|
||||
|
||||
// Watch Obsidian vault
|
||||
if w.vault != "" {
|
||||
if err := watcher.Add(w.vault); err != nil {
|
||||
log.Printf("[knox] cannot watch obsidian vault %s: %v", w.vault, err)
|
||||
} else {
|
||||
log.Printf("[knox] watching %s (obsidian)", w.vault)
|
||||
}
|
||||
}
|
||||
|
||||
// Seed existing files
|
||||
for _, ing := range w.fileIngesters {
|
||||
for _, dir := range w.dirs {
|
||||
patterns := []string{
|
||||
filepath.Join(dir, "*"),
|
||||
filepath.Join(dir, "*", "SKILL.md"),
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
entries, _ := filepath.Glob(pattern)
|
||||
for _, path := range entries {
|
||||
if !MatchesIngester(path, ing.SourceID()) {
|
||||
continue
|
||||
}
|
||||
w.ingestFileWith(path, ing, "seed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Seed Obsidian notes via the full walk: skips dot-dirs (.trash, .obsidian)
|
||||
// and covers all depths — glob patterns would match dot-dirs and miss depth >2.
|
||||
if w.vault != "" {
|
||||
if notes, err := ingest.NewObsidianIngester(w.vault).IngestAll(); err == nil {
|
||||
for _, r := range notes {
|
||||
w.recordResult(r, "seed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) ingestFile(path, trigger string) {
|
||||
for _, ing := range w.fileIngesters {
|
||||
if MatchesIngester(path, ing.SourceID()) {
|
||||
w.ingestFileWith(path, ing, trigger)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Obsidian: any .md file in the vault
|
||||
if w.vault != "" && strings.HasSuffix(path, ".md") && !strings.Contains(path, ".obsidian") {
|
||||
obs := ingest.NewObsidianFileIngester(w.vault)
|
||||
w.ingestFileWith(path, obs, trigger)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) ingestFileWith(path string, ing ingest.Ingester, trigger string) {
|
||||
result, err := ing.Ingest(path)
|
||||
if err != nil {
|
||||
log.Printf("[knox] ingest error %s: %v", path, err)
|
||||
return
|
||||
}
|
||||
if result == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Staleness check: skip when the signal time is known and unchanged.
|
||||
// Empty CreatedAt must NOT skip — it would suppress everything.
|
||||
if result.CreatedAt != "" {
|
||||
if existing, _ := w.knoxDB.FindEntry(result.Fingerprint); existing != nil && existing.CreatedAt == result.CreatedAt {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.recordResult(result, trigger)
|
||||
}
|
||||
|
||||
// recordResult persists one ingest result and fires side effects (session
|
||||
// tracking, golden-thread linking happens inside RecordObservation).
|
||||
func (w *Watcher) recordResult(result *ingest.IngestResult, trigger string) {
|
||||
obsID, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
LineStart: result.LineStart,
|
||||
LineEnd: result.LineEnd,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: trigger,
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("[knox] db error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if isNew {
|
||||
log.Printf("[knox] new entry #%d: %s", obsID, result.Title)
|
||||
} else {
|
||||
log.Printf("[knox] updated entry #%d: %s", obsID, result.Title)
|
||||
}
|
||||
|
||||
if result.SourceID == "opencode-session" {
|
||||
sessionID, _ := result.Provenance["session_id"].(string)
|
||||
if sessionID != "" {
|
||||
status := "active"
|
||||
w.knoxDB.UpsertSession(sessionID, result.Project, result.Title, status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) ingestBrowserHistory() {
|
||||
log.Printf("[knox] periodic browser history ingest...")
|
||||
ing := ingest.NewBrowserHistoryIngester()
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
log.Printf("[knox] browser ingest error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var newCount, skipped int
|
||||
for _, result := range results {
|
||||
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
|
||||
if existing != nil && existing.CreatedAt == result.CreatedAt {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "browser_timer",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
}
|
||||
}
|
||||
if newCount > 0 || skipped > 0 {
|
||||
log.Printf("[knox] browser history: %d new, %d unchanged", newCount, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) ingestGitea() {
|
||||
log.Printf("[knox] periodic gitea ingest...")
|
||||
ing := ingest.NewGiteaIngester()
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
log.Printf("[knox] gitea ingest error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var newCount, skipped int
|
||||
for _, result := range results {
|
||||
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
|
||||
if existing != nil && existing.CreatedAt == result.CreatedAt {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "gitea_timer",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
}
|
||||
}
|
||||
if newCount > 0 || skipped > 0 {
|
||||
log.Printf("[knox] gitea: %d new, %d unchanged", newCount, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) ingestGit() {
|
||||
log.Printf("[knox] periodic git status ingest...")
|
||||
ing := ingest.NewGitIngester()
|
||||
ing.Recursive = true
|
||||
if v := os.Getenv("KNOX_GIT_ROOTS"); v != "" {
|
||||
for _, part := range strings.Split(v, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
ing.Roots = append(ing.Roots, part)
|
||||
}
|
||||
}
|
||||
results, err := ing.IngestAll()
|
||||
if err != nil {
|
||||
log.Printf("[knox] git ingest error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var newCount, skipped int
|
||||
for _, result := range results {
|
||||
existing, _ := w.knoxDB.FindEntry(result.Fingerprint)
|
||||
if existing != nil && existing.CreatedAt == result.CreatedAt {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
_, isNew, err := w.knoxDB.RecordObservation(db.ObservationRecord{
|
||||
Fingerprint: result.Fingerprint,
|
||||
SourceID: result.SourceID,
|
||||
SourcePath: result.SourcePath,
|
||||
Project: result.Project,
|
||||
ContentType: result.ContentType,
|
||||
Title: result.Title,
|
||||
Summary: result.Summary,
|
||||
CreatedAt: result.CreatedAt,
|
||||
Confidence: result.Confidence,
|
||||
IngesterVersion: result.IngesterVersion,
|
||||
Trigger: "git_timer",
|
||||
Provenance: ingest.ProvenanceJSON(result.Provenance),
|
||||
})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNew {
|
||||
newCount++
|
||||
}
|
||||
}
|
||||
if newCount > 0 || skipped > 0 {
|
||||
log.Printf("[knox] git: %d new, %d unchanged", newCount, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// autoThread runs the heuristic auto-threader. Disabled entirely when
|
||||
// KNOX_THREAD_DISABLE is non-empty.
|
||||
func (w *Watcher) autoThread() {
|
||||
if os.Getenv("KNOX_THREAD_DISABLE") != "" {
|
||||
return
|
||||
}
|
||||
threader := NewAutoThreader(w.knoxDB)
|
||||
created, linked, err := threader.AutoThread()
|
||||
if err != nil {
|
||||
log.Printf("[knox] auto-thread error: %v", err)
|
||||
return
|
||||
}
|
||||
if created > 0 || linked > 0 {
|
||||
log.Printf("[knox] auto-thread: %d created, %d linked", created, linked)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) isRelevantEvent(event fsnotify.Event) bool {
|
||||
if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
|
||||
return false
|
||||
}
|
||||
name := filepath.Base(event.Name)
|
||||
// Opencode session diffs and logs
|
||||
if strings.HasPrefix(name, "ses_") || strings.HasSuffix(name, ".log") {
|
||||
return true
|
||||
}
|
||||
// Obsidian markdown files
|
||||
if w.vault != "" && strings.HasSuffix(name, ".md") && !strings.Contains(event.Name, ".obsidian") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func MatchesIngester(path, sourceID string) bool {
|
||||
base := filepath.Base(path)
|
||||
switch sourceID {
|
||||
case "opencode-session":
|
||||
return strings.HasPrefix(base, "ses_") && strings.HasSuffix(base, ".json")
|
||||
case "opencode-log":
|
||||
return strings.HasSuffix(base, ".log")
|
||||
case "skills-catalog":
|
||||
return base == "SKILL.md"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func eventOpName(op fsnotify.Op) string {
|
||||
switch {
|
||||
case op.Has(fsnotify.Create):
|
||||
return "inotify:CREATE"
|
||||
case op.Has(fsnotify.Write):
|
||||
return "inotify:WRITE"
|
||||
case op.Has(fsnotify.Chmod):
|
||||
return "inotify:CHMOD"
|
||||
default:
|
||||
return "inotify:UNKNOWN"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user