Initial commit: knox knowledge index

internal/ingest/log.go

@@ -0,0 +1,138 @@
+package ingest
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var _ Ingester = (*LogIngester)(nil)
+
+type LogIngester struct {
+	Version string
+}
+
+func NewLogIngester() *LogIngester {
+	return &LogIngester{Version: "log-parser/v1"}
+}
+
+func (l *LogIngester) SourceID() string { return "opencode-log" }
+
+var (
+	sessionIDRE = regexp.MustCompile(`session[=_ ]?([a-zA-Z0-9_-]+)`)
+	projectIDRE = regexp.MustCompile(`project[=_ ]?([a-zA-Z0-9_.-]+)`)
+	modelRE     = regexp.MustCompile(`model[=_ ]?([a-zA-Z0-9_.-]+)`)
+	agentRE     = regexp.MustCompile(`agent[=_ ]?([a-zA-Z0-9_.-]+)`)
+	errorRE     = regexp.MustCompile(`(?i)(error|fail|exception|panic|timeout)`)
+)
+
+func (l *LogIngester) Ingest(path string) (*IngestResult, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("open log: %w", err)
+	}
+	defer f.Close()
+
+	var (
+		sessionIDs []string
+		models     []string
+		agents     []string
+		errors     []string
+		totalLines int
+		firstError int
+	)
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		totalLines++
+
+		if m := sessionIDRE.FindStringSubmatch(line); len(m) > 1 {
+			sessionIDs = append(sessionIDs, m[1])
+		}
+		if m := modelRE.FindStringSubmatch(line); len(m) > 1 {
+			models = append(models, m[1])
+		}
+		if m := agentRE.FindStringSubmatch(line); len(m) > 1 {
+			agents = append(agents, m[1])
+		}
+		if errorRE.MatchString(line) {
+			if len(errors) == 0 {
+				firstError = totalLines
+			}
+			errors = append(errors, truncate(line, 120))
+		}
+	}
+
+	base := filepath.Base(path)
+	fp := Fingerprint([]byte(path))
+	title := fmt.Sprintf("Log %s (%d lines)", base, totalLines)
+	createdAt := ""
+	if fi, err := os.Stat(path); err == nil {
+		createdAt = fi.ModTime().UTC().Format(time.RFC3339)
+	}
+
+	var summaryParts []string
+	if len(sessionIDs) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("sessions: %s", uniqueJoin(sessionIDs, 5)))
+	}
+	if len(models) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("models: %s", uniqueJoin(models, 3)))
+	}
+	if len(errors) > 0 {
+		summaryParts = append(summaryParts, fmt.Sprintf("%d errors (line %d)", len(errors), firstError))
+	}
+	summary := strings.Join(summaryParts, " | ")
+	if summary == "" {
+		summary = fmt.Sprintf("%d lines", totalLines)
+	}
+
+	confidence := 0.6
+	if len(errors) > 0 {
+		confidence = 0.8
+	}
+
+	return &IngestResult{
+		Fingerprint:     fp,
+		SourceID:        l.SourceID(),
+		SourcePath:      path,
+		ContentType:     ".log",
+		Title:           title,
+		Summary:         summary,
+		CreatedAt:       createdAt,
+		Confidence:      confidence,
+		IngesterVersion: l.Version,
+		LineEnd:         totalLines,
+		Provenance: map[string]any{
+			"session_ids": uniqueSlice(sessionIDs),
+			"models":      uniqueSlice(models),
+			"agents":      uniqueSlice(agents),
+			"error_count": len(errors),
+			"file_size":   fmt.Sprintf("%d lines", totalLines),
+		},
+	}, nil
+}
+
+func uniqueJoin(items []string, max int) string {
+	uniq := uniqueSlice(items)
+	if len(uniq) > max {
+		uniq = uniq[:max]
+	}
+	return strings.Join(uniq, ", ")
+}
+
+func uniqueSlice(items []string) []string {
+	seen := make(map[string]bool)
+	var uniq []string
+	for _, s := range items {
+		if !seen[s] {
+			seen[s] = true
+			uniq = append(uniq, s)
+		}
+	}
+	return uniq
+}