Initial commit: knox knowledge index

2026-08-29 02:52:32 -07:00
commit 8eecde18b4
31 changed files with 6855 additions and 0 deletions
@@ -0,0 +1,257 @@
+package ingest
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+const (
+	chromiumHistoryPath = ".config/chromium/Default/History"
+	minHistoryPath      = ".config/Min/BrowserHistory.db"
+)
+
+// noiseURLPatterns are URLs that represent navigation noise rather than knowledge.
+var noiseURLPatterns = []string{
+	"opencode.ai",
+	"auth.opencode.ai",
+	"github.com/login/oauth/authorize",
+	"accounts.google.com",
+	"google.com/accounts",
+	"kagi.com",
+	"chatgpt.com",
+	// knox indexing its own web UI is self-referential noise
+	"localhost:8924",
+	"localhost:18925",
+	"127.0.0.1:8924",
+	"127.0.0.1:18925",
+}
+
+func noiseURL(url string) bool {
+	for _, p := range noiseURLPatterns {
+		if strings.Contains(url, p) {
+			return true
+		}
+	}
+	return false
+}
+
+type BrowserHistoryIngester struct {
+	Version string
+}
+
+func NewBrowserHistoryIngester() *BrowserHistoryIngester {
+	return &BrowserHistoryIngester{Version: "browser-history/v1"}
+}
+
+func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" }
+
+type historyRow struct {
+	URL        string
+	Title      string
+	VisitTime  time.Time
+	VisitCount int
+}
+
+func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) {
+	home, _ := os.UserHomeDir()
+	var results []*IngestResult
+
+	// Chromium
+	chromiumDB := filepath.Join(home, chromiumHistoryPath)
+	cr, err := b.ingestChromium(chromiumDB)
+	if err == nil {
+		results = append(results, cr...)
+	}
+
+	// Min
+	minDB := filepath.Join(home, minHistoryPath)
+	minResults, err := b.ingestMin(minDB)
+	if err == nil {
+		results = append(results, minResults...)
+	}
+
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) {
+	if !fileExists(path) {
+		return nil, nil
+	}
+
+	// Copy to avoid lock issues (Chromium uses WAL)
+	tmp := path + ".knox_tmp"
+	if err := copyFile(path, tmp); err != nil {
+		return nil, fmt.Errorf("copy chromium db: %w", err)
+	}
+	defer os.Remove(tmp)
+
+	db, err := sql.Open("sqlite", tmp)
+	if err != nil {
+		return nil, fmt.Errorf("open chromium copy: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(
+		`SELECT url, COALESCE(title,''), last_visit_time, visit_count
+		 FROM urls
+		 WHERE last_visit_time > 0
+		 ORDER BY last_visit_time DESC
+		 LIMIT 1000`,
+	)
+	if err != nil {
+		// Table might not exist or schema mismatch
+		return nil, fmt.Errorf("query chromium urls: %w", err)
+	}
+	defer rows.Close()
+
+	var results []*IngestResult
+	for rows.Next() {
+		var url, title string
+		var visitTimeMicro int64
+		var visitCount int
+		if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil {
+			continue
+		}
+
+		// Chromium webkit time: microseconds since 1601-01-01 UTC
+		visitTime := webkitToTime(visitTimeMicro)
+
+		if result := b.makeResult(url, title, visitTime, visitCount); result != nil {
+			results = append(results, result)
+		}
+	}
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) {
+	if !fileExists(path) {
+		return nil, nil
+	}
+
+	tmp := path + ".knox_tmp"
+	if err := copyFile(path, tmp); err != nil {
+		return nil, fmt.Errorf("copy min db: %w", err)
+	}
+	defer os.Remove(tmp)
+
+	db, err := sql.Open("sqlite", tmp)
+	if err != nil {
+		return nil, fmt.Errorf("open min copy: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.Query(
+		`SELECT url, COALESCE(title,''), timestamp
+		 FROM history
+		 ORDER BY timestamp DESC
+		 LIMIT 1000`,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("query min history: %w", err)
+	}
+	defer rows.Close()
+
+	var results []*IngestResult
+	for rows.Next() {
+		var url, title string
+		var ts int64
+		if err := rows.Scan(&url, &title, &ts); err != nil {
+			continue
+		}
+
+		visitTime := time.Unix(ts, 0)
+		if result := b.makeResult(url, title, visitTime, 1); result != nil {
+			results = append(results, result)
+		}
+	}
+	return results, nil
+}
+
+func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult {
+	if url == "" {
+		return nil
+	}
+
+	// Skip chrome://, about://, file://, devtools
+	if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") ||
+		strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") ||
+		strings.HasPrefix(url, "chrome-extension://") {
+		return nil
+	}
+
+	if noiseURL(url) {
+		return nil
+	}
+
+	displayTitle := title
+	if displayTitle == "" {
+		displayTitle = extractDomain(url)
+	}
+
+	fp := Fingerprint([]byte(url))
+
+	confidence := 0.6
+	if visitCount > 5 {
+		confidence = 0.9
+	} else if visitCount > 1 {
+		confidence = 0.7
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        b.SourceID(),
+		SourcePath:      url,
+		ContentType:     "url",
+		Title:           displayTitle,
+		Summary:         url,
+		CreatedAt:       visitTime.Format(time.RFC3339),
+		Confidence:      confidence,
+		IngesterVersion: b.Version,
+		Provenance: map[string]any{
+			"url":         url,
+			"visit_count": visitCount,
+			"visited_at":  visitTime.Format(time.RFC3339),
+		},
+	}
+}
+
+// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time
+func webkitToTime(micros int64) time.Time {
+	// WebKit epoch: 1601-01-01 UTC
+	// Unix epoch: 1970-01-01 UTC
+	// Difference: 11644473600 seconds
+	secs := micros / 1_000_000
+	if secs < 11644473600 {
+		return time.Time{}
+	}
+	return time.Unix(secs-11644473600, 0)
+}
+
+func extractDomain(url string) string {
+	url = strings.TrimPrefix(url, "https://")
+	url = strings.TrimPrefix(url, "http://")
+	parts := strings.Split(url, "/")
+	if len(parts) > 0 {
+		return parts[0]
+	}
+	return url
+}
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func copyFile(src, dst string) error {
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(dst, data, 0644)
+}