Initial commit: knox knowledge index

2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
@@ -0,0 +1,135 @@
+package cmd
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/david/knox/internal/db"
+	"github.com/david/knox/internal/index"
+)
+
+func NewTopicsCmd(kdb *db.KnoxDB) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "topics",
+		Short: "Auto-detect topic clusters from TF-IDF analysis of the knowledge index",
+		Long: `Scans all entries, computes TF-IDF term scores, and clusters related
+observations into topics. Shows gap analysis for browser-researched
+topics not covered by skills.
+
+Subcommands: list, gaps`,
+	}
+
+	cmd.AddCommand(newTopicListCmd(kdb))
+	cmd.AddCommand(newTopicGapsCmd(kdb))
+
+	return cmd
+}
+
+func newTopicListCmd(kdb *db.KnoxDB) *cobra.Command {
+	return &cobra.Command{
+		Use:   "list",
+		Short: "List auto-detected topic clusters",
+		RunE: func(c *cobra.Command, args []string) error {
+			entries, err := kdb.RecentEntries(2000)
+			if err != nil {
+				return err
+			}
+
+			tfidf := index.BuildTFIDF(entries)
+			clusters := tfidf.Cluster(2, 15)
+
+			if len(clusters) == 0 {
+				fmt.Println("No topic clusters detected.")
+				return nil
+			}
+
+			fmt.Printf("%-3s %-50s %-8s %s\n", "ID", "Topic", "Entries", "Sources")
+			fmt.Println(strings.Repeat("-", 75))
+			for i, c := range clusters {
+				total := 0
+				for _, v := range c.BySource {
+					total += v
+				}
+				sources := ""
+				for src, count := range c.BySource {
+					if sources != "" {
+						sources += ", "
+					}
+					sources += fmt.Sprintf("%s:%d", shortSource(src), count)
+				}
+				fmt.Printf("%-3d %-50s %-8d %s\n", i+1, truncateStr(c.Name, 48), total, sources)
+			}
+			return nil
+		},
+	}
+}
+
+func newTopicGapsCmd(kdb *db.KnoxDB) *cobra.Command {
+	return &cobra.Command{
+		Use:   "gaps",
+		Short: "Show topics with browser activity but no skill coverage",
+		Long: `Cross-references browser history against the skills catalog.
+Flags topics that have significant browser research but zero skill entries.
+
+These are candidates for new skill creation.`,
+		RunE: func(c *cobra.Command, args []string) error {
+			entries, err := kdb.RecentEntries(2000)
+			if err != nil {
+				return err
+			}
+
+			// Build skill keyword set
+			skillKeywords := make(map[string]bool)
+			for _, e := range entries {
+				if e.SourceID == "skills-catalog" {
+					for _, w := range tokenize(e.Title + " " + e.Summary) {
+						skillKeywords[w] = true
+					}
+				}
+			}
+
+			tfidf := index.BuildTFIDF(entries)
+			clusters := tfidf.Cluster(2, 15)
+			gaps := tfidf.Gaps(clusters, skillKeywords)
+
+			if len(gaps) == 0 {
+				fmt.Println("No gaps found — all browser-researched topics have skill coverage.")
+				return nil
+			}
+
+			fmt.Printf("\nFound %d topics with browser activity but NO skill coverage:\n\n", len(gaps))
+			for i, g := range gaps {
+				browserCount := g.BySource["browser-history"]
+				fmt.Printf("%d. %s\n", i+1, g.Name)
+				fmt.Printf("   %d browser visits · keywords: %v\n", browserCount, g.Keywords)
+				fmt.Println()
+			}
+
+			fmt.Println("Consider creating skills for these topics with `knox asses` or via opencode.")
+			return nil
+		},
+	}
+}
+
+func shortSource(s string) string {
+	switch s {
+	case "opencode-session":
+		return "session"
+	case "skills-catalog":
+		return "skill"
+	case "browser-history":
+		return "browser"
+	case "obsidian":
+		return "notes"
+	case "opencode-log":
+		return "logs"
+	}
+	return s
+}
+
+func tokenize(text string) []string {
+	re := regexp.MustCompile(`[a-z]{4,}`)
+	return re.FindAllString(strings.ToLower(text), -1)
+}