knox/internal/ingest/browser.go

package ingest

import (
	"database/sql"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	_ "modernc.org/sqlite"
)

const (
	chromiumHistoryPath = ".config/chromium/Default/History"
	minHistoryPath      = ".config/Min/BrowserHistory.db"
)

// noiseURLPatterns are URLs that represent navigation noise rather than knowledge.
var noiseURLPatterns = []string{
	"opencode.ai",
	"auth.opencode.ai",
	"github.com/login/oauth/authorize",
	"accounts.google.com",
	"google.com/accounts",
	"kagi.com",
	"chatgpt.com",
	// knox indexing its own web UI is self-referential noise
	"localhost:8924",
	"localhost:18925",
	"127.0.0.1:8924",
	"127.0.0.1:18925",
}

func noiseURL(url string) bool {
	for _, p := range noiseURLPatterns {
		if strings.Contains(url, p) {
			return true
		}
	}
	return false
}

type BrowserHistoryIngester struct {
	Version string
}

func NewBrowserHistoryIngester() *BrowserHistoryIngester {
	return &BrowserHistoryIngester{Version: "browser-history/v1"}
}

func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" }

type historyRow struct {
	URL        string
	Title      string
	VisitTime  time.Time
	VisitCount int
}

func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) {
	home, _ := os.UserHomeDir()
	var results []*IngestResult

	// Chromium
	chromiumDB := filepath.Join(home, chromiumHistoryPath)
	cr, err := b.ingestChromium(chromiumDB)
	if err == nil {
		results = append(results, cr...)
	}

	// Min
	minDB := filepath.Join(home, minHistoryPath)
	minResults, err := b.ingestMin(minDB)
	if err == nil {
		results = append(results, minResults...)
	}

	return results, nil
}

func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) {
	if !fileExists(path) {
		return nil, nil
	}

	// Copy to avoid lock issues (Chromium uses WAL)
	tmp := path + ".knox_tmp"
	if err := copyFile(path, tmp); err != nil {
		return nil, fmt.Errorf("copy chromium db: %w", err)
	}
	defer os.Remove(tmp)

	db, err := sql.Open("sqlite", tmp)
	if err != nil {
		return nil, fmt.Errorf("open chromium copy: %w", err)
	}
	defer db.Close()

	rows, err := db.Query(
		`SELECT url, COALESCE(title,''), last_visit_time, visit_count
		 FROM urls
		 WHERE last_visit_time > 0
		 ORDER BY last_visit_time DESC
		 LIMIT 1000`,
	)
	if err != nil {
		// Table might not exist or schema mismatch
		return nil, fmt.Errorf("query chromium urls: %w", err)
	}
	defer rows.Close()

	var results []*IngestResult
	for rows.Next() {
		var url, title string
		var visitTimeMicro int64
		var visitCount int
		if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil {
			continue
		}

		// Chromium webkit time: microseconds since 1601-01-01 UTC
		visitTime := webkitToTime(visitTimeMicro)

		if result := b.makeResult(url, title, visitTime, visitCount); result != nil {
			results = append(results, result)
		}
	}
	return results, nil
}

func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) {
	if !fileExists(path) {
		return nil, nil
	}

	tmp := path + ".knox_tmp"
	if err := copyFile(path, tmp); err != nil {
		return nil, fmt.Errorf("copy min db: %w", err)
	}
	defer os.Remove(tmp)

	db, err := sql.Open("sqlite", tmp)
	if err != nil {
		return nil, fmt.Errorf("open min copy: %w", err)
	}
	defer db.Close()

	rows, err := db.Query(
		`SELECT url, COALESCE(title,''), timestamp
		 FROM history
		 ORDER BY timestamp DESC
		 LIMIT 1000`,
	)
	if err != nil {
		return nil, fmt.Errorf("query min history: %w", err)
	}
	defer rows.Close()

	var results []*IngestResult
	for rows.Next() {
		var url, title string
		var ts int64
		if err := rows.Scan(&url, &title, &ts); err != nil {
			continue
		}

		visitTime := time.Unix(ts, 0)
		if result := b.makeResult(url, title, visitTime, 1); result != nil {
			results = append(results, result)
		}
	}
	return results, nil
}

func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult {
	if url == "" {
		return nil
	}

	// Skip chrome://, about://, file://, devtools
	if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") ||
		strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") ||
		strings.HasPrefix(url, "chrome-extension://") {
		return nil
	}

	if noiseURL(url) {
		return nil
	}

	displayTitle := title
	if displayTitle == "" {
		displayTitle = extractDomain(url)
	}

	fp := Fingerprint([]byte(url))

	confidence := 0.6
	if visitCount > 5 {
		confidence = 0.9
	} else if visitCount > 1 {
		confidence = 0.7
	}

	return &IngestResult{
		Fingerprint:     fp,
		SourceID:        b.SourceID(),
		SourcePath:      url,
		ContentType:     "url",
		Title:           displayTitle,
		Summary:         url,
		CreatedAt:       visitTime.Format(time.RFC3339),
		Confidence:      confidence,
		IngesterVersion: b.Version,
		Provenance: map[string]any{
			"url":         url,
			"visit_count": visitCount,
			"visited_at":  visitTime.Format(time.RFC3339),
		},
	}
}

// webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time
func webkitToTime(micros int64) time.Time {
	// WebKit epoch: 1601-01-01 UTC
	// Unix epoch: 1970-01-01 UTC
	// Difference: 11644473600 seconds
	secs := micros / 1_000_000
	if secs < 11644473600 {
		return time.Time{}
	}
	return time.Unix(secs-11644473600, 0)
}

func extractDomain(url string) string {
	url = strings.TrimPrefix(url, "https://")
	url = strings.TrimPrefix(url, "http://")
	parts := strings.Split(url, "/")
	if len(parts) > 0 {
		return parts[0]
	}
	return url
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func copyFile(src, dst string) error {
	data, err := os.ReadFile(src)
	if err != nil {
		return err
	}
	return os.WriteFile(dst, data, 0644)
}