knox/internal/cmd/ingest.go

package cmd

import (
	"fmt"
	"log"
	"path/filepath"

	"github.com/david/knox/internal/db"
	"github.com/david/knox/internal/ingest"
	"github.com/david/knox/internal/watch"
	"github.com/spf13/cobra"
)

func NewIngestCmd(kdb *db.KnoxDB) *cobra.Command {
	var dirs []string
	cmd := &cobra.Command{
		Use:   "ingest",
		Short: "One-time full sweep of all sources into the knowledge index",
		Run: func(cmd *cobra.Command, args []string) {
			if len(dirs) == 0 {
				dirs = defaultWatchDirs()
			}

			ingesters := []ingest.Ingester{
				ingest.NewSessionDiffIngester(),
				ingest.NewLogIngester(),
				ingest.NewSkillsIngester(),
			}

			var totalNew, totalUpdated int

			record := func(result *ingest.IngestResult) {
				_, isNew, err := kdb.RecordObservation(db.ObservationRecord{
					Fingerprint:     result.Fingerprint,
					SourceID:        result.SourceID,
					SourcePath:      result.SourcePath,
					Project:         result.Project,
					ContentType:     result.ContentType,
					Title:           result.Title,
					Summary:         result.Summary,
					CreatedAt:       result.CreatedAt,
					LineStart:       result.LineStart,
					LineEnd:         result.LineEnd,
					Confidence:      result.Confidence,
					IngesterVersion: result.IngesterVersion,
					Trigger:         "ingest",
					Provenance:      ingest.ProvenanceJSON(result.Provenance),
				})
				if err != nil {
					log.Printf("[knox] db error: %v", err)
					return
				}
				if isNew {
					totalNew++
				} else {
					totalUpdated++
				}

				if result.SourceID == "opencode-session" {
					sessionID, _ := result.Provenance["session_id"].(string)
					if sessionID != "" {
						kdb.UpsertSession(sessionID, result.Project, result.Title, "active")
					}
				}
			}

			for _, ing := range ingesters {
				sid := ing.SourceID()
				log.Printf("[knox] ingesting %s...", sid)

				for _, dir := range dirs {
					patterns := []string{
						dir + "/*",
						dir + "/*/SKILL.md",
					}
					for _, pattern := range patterns {
						entries, _ := filepath.Glob(pattern)
						for _, path := range entries {
							if !watch.MatchesIngester(path, sid) {
								continue
							}

							result, err := ing.Ingest(path)
							if err != nil || result == nil {
								continue
							}
							record(result)
						}
					}
				}
			}

			// Non-file sources: obsidian vault, browser history, gitea
			if vault, err := ingest.DetectObsidianVault(); err == nil {
				log.Printf("[knox] ingesting obsidian...")
				if results, err := ingest.NewObsidianIngester(vault).IngestAll(); err == nil {
					for _, r := range results {
						record(r)
					}
				} else {
					log.Printf("[knox] obsidian ingest failed: %v", err)
				}
			}
			log.Printf("[knox] ingesting browser-history...")
			if results, err := ingest.NewBrowserHistoryIngester().IngestAll(); err == nil {
				for _, r := range results {
					record(r)
				}
			} else {
				log.Printf("[knox] browser-history ingest failed: %v", err)
			}
			log.Printf("[knox] ingesting gitea...")
			if results, err := ingest.NewGiteaIngester().IngestAll(); err == nil {
				for _, r := range results {
					record(r)
				}
			} else {
				log.Printf("[knox] gitea ingest failed: %v", err)
			}

			fmt.Printf("\nIngest complete: %d new entries, %d updated\n", totalNew, totalUpdated)
		},
	}
	cmd.Flags().StringSliceVarP(&dirs, "dir", "d", nil, "Directories to scan (default: opencode storage/log + skills)")
	return cmd
}