Initial commit: knox knowledge index
This commit is contained in:
@@ -0,0 +1,269 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GitIngester tracks project status for local git repositories.
|
||||
//
|
||||
// For each repo under one or more scan roots (default ~/src) it records the
|
||||
// current branch, whether the working tree is clean, ahead/behind counts vs
|
||||
// the upstream, the HEAD hash, and the last commit time + subject. The entry's
|
||||
// CreatedAt is the last commit time, so staleness/dedup works naturally and
|
||||
// an entry updates when new commits land.
|
||||
//
|
||||
// When Recursive is set, repos are discovered at any depth under a root
|
||||
// (following symlinked directories), not just direct children — this picks up
|
||||
// nested repos like ~/assistant/kitmaker or ~/Arduino/*/... Repos already have
|
||||
// a stable identity via their real (symlink-resolved) path, so overlapping
|
||||
// roots across symlinks do not duplicate entries.
|
||||
type GitIngester struct {
|
||||
Version string
|
||||
Root string // single scan root; overridden by Roots (empty => $HOME/src)
|
||||
Roots []string // multi-scan roots; overrides Root when non-empty
|
||||
Recursive bool // recurse into subdirs (and follow dir symlinks) to find nested .git
|
||||
}
|
||||
|
||||
func NewGitIngester() *GitIngester {
|
||||
return &GitIngester{Version: "git/v1"}
|
||||
}
|
||||
|
||||
func (g *GitIngester) SourceID() string { return "git" }
|
||||
|
||||
// scanRoots returns the effective list of scan roots.
|
||||
func (g *GitIngester) scanRoots() []string {
|
||||
if len(g.Roots) > 0 {
|
||||
return g.Roots
|
||||
}
|
||||
if g.Root != "" {
|
||||
return []string{g.Root}
|
||||
}
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return []string{filepath.Join(home, "src")}
|
||||
}
|
||||
|
||||
func (g *GitIngester) IngestAll() ([]*IngestResult, error) {
|
||||
// Dedup by real path so overlapping roots / symlinked roots don't double-add.
|
||||
seen := make(map[string]struct{})
|
||||
var results []*IngestResult
|
||||
for _, root := range g.scanRoots() {
|
||||
for _, repoPath := range collectGitRepos(root, g.Recursive) {
|
||||
real, err := filepath.EvalSymlinks(repoPath)
|
||||
if err != nil {
|
||||
real = repoPath
|
||||
}
|
||||
if _, dup := seen[real]; dup {
|
||||
continue
|
||||
}
|
||||
seen[real] = struct{}{}
|
||||
if r := g.repoToResult(real); r != nil {
|
||||
results = append(results, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func isGitRepo(path string) bool {
|
||||
info, err := os.Stat(filepath.Join(path, ".git"))
|
||||
return err == nil && info.IsDir()
|
||||
}
|
||||
|
||||
// collectGitRepos returns paths of git repos under root. In non-recursive mode
|
||||
// only direct children are considered (root itself is not counted), matching
|
||||
// the original behavior. In recursive mode the root is walked to any depth,
|
||||
// descending into symlinked directories, up to and including the root if it is
|
||||
// itself a git repo. Descending stops once a git repo is found, so nested
|
||||
// repos (e.g. a vendored copy inside another repo) are treated as leaves.
|
||||
func collectGitRepos(root string, recursive bool) []string {
|
||||
visited := make(map[string]struct{})
|
||||
var out []string
|
||||
|
||||
var walk func(dir string)
|
||||
walk = func(dir string) {
|
||||
real, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := visited[real]; ok {
|
||||
return
|
||||
}
|
||||
visited[real] = struct{}{}
|
||||
|
||||
if isGitRepo(real) {
|
||||
out = append(out, real)
|
||||
return
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(real)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
child := filepath.Join(real, e.Name())
|
||||
if recursive {
|
||||
walk(child)
|
||||
continue
|
||||
}
|
||||
if e.IsDir() && isGitRepo(child) {
|
||||
out = append(out, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(root)
|
||||
return out
|
||||
}
|
||||
|
||||
type GitStatus struct {
|
||||
Branch string
|
||||
Head string
|
||||
Dirty int
|
||||
Ahead int
|
||||
Behind int
|
||||
LastCommit string
|
||||
Subject string
|
||||
}
|
||||
|
||||
func (g *GitIngester) repoToResult(repoPath string) *IngestResult {
|
||||
status := gitStatus(repoPath)
|
||||
name := filepath.Base(repoPath)
|
||||
fp := Fingerprint([]byte("git:repo:" + repoPath))
|
||||
|
||||
clean := "clean"
|
||||
if status.Dirty > 0 {
|
||||
clean = fmt.Sprintf("%d change(s)", status.Dirty)
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("%s @ %s — %s", name, status.Branch, clean)
|
||||
if status.Ahead > 0 || status.Behind > 0 {
|
||||
summary += fmt.Sprintf(", ahead %d / behind %d", status.Ahead, status.Behind)
|
||||
}
|
||||
if status.Subject != "" {
|
||||
summary += " — “" + truncate(status.Subject, 80) + "”"
|
||||
}
|
||||
|
||||
createdAt := status.LastCommit
|
||||
if createdAt == "" {
|
||||
createdAt = gitStatusTime(status)
|
||||
}
|
||||
|
||||
return &IngestResult{
|
||||
Fingerprint: fp,
|
||||
SourceID: g.SourceID(),
|
||||
SourcePath: repoPath,
|
||||
Project: name,
|
||||
ContentType: "git-status",
|
||||
Title: name,
|
||||
Summary: summary,
|
||||
CreatedAt: createdAt,
|
||||
Confidence: 0.9,
|
||||
IngesterVersion: g.Version,
|
||||
Provenance: map[string]any{
|
||||
"type": "git_status",
|
||||
"path": repoPath,
|
||||
"branch": status.Branch,
|
||||
"head": status.Head,
|
||||
"dirty": status.Dirty,
|
||||
"ahead": status.Ahead,
|
||||
"behind": status.Behind,
|
||||
"last_commit": status.LastCommit,
|
||||
"subject": status.Subject,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func gitStatusTime(s GitStatus) string {
|
||||
if s.LastCommit != "" {
|
||||
return s.LastCommit
|
||||
}
|
||||
return time.Now().UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func gitStatus(repoPath string) GitStatus {
|
||||
out, err := gitOut(repoPath, "status", "--porcelain")
|
||||
if err != nil {
|
||||
return GitStatus{}
|
||||
}
|
||||
dirty := 0
|
||||
if out != "" {
|
||||
dirty = len(strings.Split(strings.TrimRight(out, "\n"), "\n"))
|
||||
}
|
||||
|
||||
branch, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
head, _ := gitOut(repoPath, "rev-parse", "--short", "HEAD")
|
||||
lastCommit, _ := gitOut(repoPath, "log", "-1", "--format=%cI")
|
||||
subject, _ := gitOut(repoPath, "log", "-1", "--format=%s")
|
||||
|
||||
ahead, behind := 0, 0
|
||||
// ahead/behind only meaningful if there's an upstream
|
||||
if upOut, _ := gitOut(repoPath, "rev-parse", "--abbrev-ref", "HEAD@{upstream}"); upOut != "" {
|
||||
if rev, err := gitOut(repoPath, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err == nil {
|
||||
fields := strings.Fields(rev)
|
||||
if len(fields) == 2 {
|
||||
var a, b int
|
||||
if _, e1 := fmt.Sscanf(fields[0], "%d", &a); e1 == nil {
|
||||
ahead = a
|
||||
}
|
||||
if _, e2 := fmt.Sscanf(fields[1], "%d", &b); e2 == nil {
|
||||
behind = b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return GitStatus{
|
||||
Branch: strings.TrimSpace(branch),
|
||||
Head: strings.TrimSpace(head),
|
||||
Dirty: dirty,
|
||||
Ahead: ahead,
|
||||
Behind: behind,
|
||||
LastCommit: strings.TrimSpace(lastCommit),
|
||||
Subject: strings.TrimSpace(subject),
|
||||
}
|
||||
}
|
||||
|
||||
// sanitizeGitField strips ANSI codes and control bytes from git output
|
||||
// (colour/unicode modes can inject escape sequences).
|
||||
var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)
|
||||
|
||||
func sanitizeGitField(s string) string {
|
||||
s = ansiRe.ReplaceAllString(s, "")
|
||||
s = strings.ReplaceAll(s, "\x1b", "")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// gitOut runs git in repoPath and returns trimmed stdout (or "" on error).
|
||||
func gitOut(repoPath string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", append([]string{"-C", repoPath}, args...)...)
|
||||
cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sanitizeGitField(string(out)), nil
|
||||
}
|
||||
|
||||
func (g *GitIngester) Ingest(_ string) (*IngestResult, error) {
|
||||
return nil, fmt.Errorf("use IngestAll() for git")
|
||||
}
|
||||
|
||||
var _ Ingester = (*GitIngester)(nil)
|
||||
|
||||
// SortResults orders results deterministically by project name (helper for CLI output).
|
||||
func SortResults(res []*IngestResult) {
|
||||
sort.Slice(res, func(i, j int) bool { return res[i].Project < res[j].Project })
|
||||
}
|
||||
Reference in New Issue
Block a user