Initial commit: knox knowledge index

2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
@@ -0,0 +1,120 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/david/knox/internal/db"
+	"github.com/david/knox/internal/ingest"
+	"github.com/david/knox/internal/watch"
+)
+
+func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
+	var dirs []string
+	cmd := &cobra.Command{
+		Use:   "ingest",
+		Short: "One-time full sweep of all sources into the knowledge index",
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(dirs) == 0 {
+				dirs = defaultWatchDirs()
+			}
+
+			ingesters := []ingest.Ingester{
+				ingest.NewSessionDiffIngester(),
+				ingest.NewLogIngester(),
+				ingest.NewSkillsIngester(),
+			}
+
+		var totalNew, totalUpdated int
+
+		record := func(result *ingest.IngestResult) {
+			_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
+				Fingerprint:     result.Fingerprint,
+				SourceID:        result.SourceID,
+				SourcePath:      result.SourcePath,
+				Project:         result.Project,
+				ContentType:     result.ContentType,
+				Title:           result.Title,
+				Summary:         result.Summary,
+				CreatedAt:       result.CreatedAt,
+				LineStart:       result.LineStart,
+				LineEnd:         result.LineEnd,
+				Confidence:      result.Confidence,
+				IngesterVersion: result.IngesterVersion,
+				Trigger:         "ingest",
+				Provenance:      ingest.ProvenanceJSON(result.Provenance),
+			})
+			if err != nil {
+				log.Printf("[knox] db error: %v", err)
+				return
+			}
+			if isNew {
+				totalNew++
+			} else {
+				totalUpdated++
+			}
+
+			if result.SourceID == "opencode-session" {
+				sessionID, _ := result.Provenance["session_id"].(string)
+				if sessionID != "" {
+					kdb.UpsertSession(sessionID, result.Project, result.Title, "active")
+				}
+			}
+		}
+
+		for _, ing := range ingesters {
+			sid := ing.SourceID()
+			log.Printf("[knox] ingesting %s...", sid)
+
+			for _, dir := range dirs {
+				patterns := []string{
+					dir + "/*",
+					dir + "/*/SKILL.md",
+				}
+				for _, pattern := range patterns {
+					entries, _ := filepath.Glob(pattern)
+					for _, path := range entries {
+						if !watch.MatchesIngester(path, sid) {
+							continue
+						}
+
+						result, err := ing.Ingest(path)
+						if err != nil || result == nil {
+							continue
+						}
+						record(result)
+					}
+				}
+			}
+		}
+
+		// Non-file sources: obsidian vault, browser history, gitea
+		if vault, err := ingest.DetectObsidianVault(); err == nil {
+			log.Printf("[knox] ingesting obsidian...")
+			if results, err := ingest.NewObsidianIngester(vault).IngestAll(); err == nil {
+				for _, r := range results {
+					record(r)
+				}
+			}
+		}
+		log.Printf("[knox] ingesting browser-history...")
+		if results, err := ingest.NewBrowserHistoryIngester().IngestAll(); err == nil {
+			for _, r := range results {
+				record(r)
+			}
+		}
+		log.Printf("[knox] ingesting gitea...")
+		if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
+			for _, r := range results {
+				record(r)
+			}
+		}
+
+		fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
+		},
+	}
+	cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to scan (default: opencode storage/log + skills)")
+	return cmd
+}