package ingest import ( "database/sql" "fmt" "os" "path/filepath" "strings" "time" _ "modernc.org/sqlite" ) const ( chromiumHistoryPath = ".config/chromium/Default/History" minHistoryPath = ".config/Min/BrowserHistory.db" ) // noiseURLPatterns are URLs that represent navigation noise rather than knowledge. var noiseURLPatterns = []string{ "opencode.ai", "auth.opencode.ai", "github.com/login/oauth/authorize", "accounts.google.com", "google.com/accounts", "kagi.com", "chatgpt.com", // knox indexing its own web UI is self-referential noise "localhost:8924", "localhost:18925", "127.0.0.1:8924", "127.0.0.1:18925", } func noiseURL(url string) bool { for _, p := range noiseURLPatterns { if strings.Contains(url, p) { return true } } return false } type BrowserHistoryIngester struct { Version string } func NewBrowserHistoryIngester() *BrowserHistoryIngester { return &BrowserHistoryIngester{Version: "browser-history/v1"} } func (b *BrowserHistoryIngester) SourceID() string { return "browser-history" } type historyRow struct { URL string Title string VisitTime time.Time VisitCount int } func (b *BrowserHistoryIngester) IngestAll() ([]*IngestResult, error) { home, _ := os.UserHomeDir() var results []*IngestResult // Chromium chromiumDB := filepath.Join(home, chromiumHistoryPath) cr, err := b.ingestChromium(chromiumDB) if err == nil { results = append(results, cr...) } // Min minDB := filepath.Join(home, minHistoryPath) minResults, err := b.ingestMin(minDB) if err == nil { results = append(results, minResults...) } return results, nil } func (b *BrowserHistoryIngester) ingestChromium(path string) ([]*IngestResult, error) { if !fileExists(path) { return nil, nil } // Copy to avoid lock issues (Chromium uses WAL) tmp := path + ".knox_tmp" if err := copyFile(path, tmp); err != nil { return nil, fmt.Errorf("copy chromium db: %w", err) } defer os.Remove(tmp) db, err := sql.Open("sqlite", tmp) if err != nil { return nil, fmt.Errorf("open chromium copy: %w", err) } defer db.Close() rows, err := db.Query( `SELECT url, COALESCE(title,''), last_visit_time, visit_count FROM urls WHERE last_visit_time > 0 ORDER BY last_visit_time DESC LIMIT 1000`, ) if err != nil { // Table might not exist or schema mismatch return nil, fmt.Errorf("query chromium urls: %w", err) } defer rows.Close() var results []*IngestResult for rows.Next() { var url, title string var visitTimeMicro int64 var visitCount int if err := rows.Scan(&url, &title, &visitTimeMicro, &visitCount); err != nil { continue } // Chromium webkit time: microseconds since 1601-01-01 UTC visitTime := webkitToTime(visitTimeMicro) if result := b.makeResult(url, title, visitTime, visitCount); result != nil { results = append(results, result) } } return results, nil } func (b *BrowserHistoryIngester) ingestMin(path string) ([]*IngestResult, error) { if !fileExists(path) { return nil, nil } tmp := path + ".knox_tmp" if err := copyFile(path, tmp); err != nil { return nil, fmt.Errorf("copy min db: %w", err) } defer os.Remove(tmp) db, err := sql.Open("sqlite", tmp) if err != nil { return nil, fmt.Errorf("open min copy: %w", err) } defer db.Close() rows, err := db.Query( `SELECT url, COALESCE(title,''), timestamp FROM history ORDER BY timestamp DESC LIMIT 1000`, ) if err != nil { return nil, fmt.Errorf("query min history: %w", err) } defer rows.Close() var results []*IngestResult for rows.Next() { var url, title string var ts int64 if err := rows.Scan(&url, &title, &ts); err != nil { continue } visitTime := time.Unix(ts, 0) if result := b.makeResult(url, title, visitTime, 1); result != nil { results = append(results, result) } } return results, nil } func (b *BrowserHistoryIngester) makeResult(url, title string, visitTime time.Time, visitCount int) *IngestResult { if url == "" { return nil } // Skip chrome://, about://, file://, devtools if strings.HasPrefix(url, "chrome://") || strings.HasPrefix(url, "about:") || strings.HasPrefix(url, "file://") || strings.HasPrefix(url, "devtools://") || strings.HasPrefix(url, "chrome-extension://") { return nil } if noiseURL(url) { return nil } displayTitle := title if displayTitle == "" { displayTitle = extractDomain(url) } fp := Fingerprint([]byte(url)) confidence := 0.6 if visitCount > 5 { confidence = 0.9 } else if visitCount > 1 { confidence = 0.7 } return &IngestResult{ Fingerprint: fp, SourceID: b.SourceID(), SourcePath: url, ContentType: "url", Title: displayTitle, Summary: url, CreatedAt: visitTime.Format(time.RFC3339), Confidence: confidence, IngesterVersion: b.Version, Provenance: map[string]any{ "url": url, "visit_count": visitCount, "visited_at": visitTime.Format(time.RFC3339), }, } } // webkitToTime converts Chromium webkit timestamp (µs since 1601-01-01) to time.Time func webkitToTime(micros int64) time.Time { // WebKit epoch: 1601-01-01 UTC // Unix epoch: 1970-01-01 UTC // Difference: 11644473600 seconds secs := micros / 1_000_000 if secs < 11644473600 { return time.Time{} } return time.Unix(secs-11644473600, 0) } func extractDomain(url string) string { url = strings.TrimPrefix(url, "https://") url = strings.TrimPrefix(url, "http://") parts := strings.Split(url, "/") if len(parts) > 0 { return parts[0] } return url } func fileExists(path string) bool { _, err := os.Stat(path) return err == nil } func copyFile(src, dst string) error { data, err := os.ReadFile(src) if err != nil { return err } return os.WriteFile(dst, data, 0644) }