58 lines
1.0 KiB
Go
58 lines
1.0 KiB
Go
package ingest
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
)
|
|
|
|
type IngestResult struct {
|
|
Fingerprint string
|
|
SourceID string
|
|
SourcePath string
|
|
Project string
|
|
ContentType string
|
|
Title string
|
|
Summary string
|
|
CreatedAt string
|
|
LineStart int
|
|
LineEnd int
|
|
Confidence float64
|
|
IngesterVersion string
|
|
Provenance map[string]any
|
|
}
|
|
|
|
type Ingester interface {
|
|
SourceID() string
|
|
Ingest(path string) (*IngestResult, error)
|
|
}
|
|
|
|
func Fingerprint(data []byte) string {
|
|
h := sha256.Sum256(data)
|
|
return fmt.Sprintf("%x", h[:16])
|
|
}
|
|
|
|
func FingerprintWithMeta(data []byte, meta map[string]string) string {
|
|
h := sha256.New()
|
|
h.Write(data)
|
|
enc := json.NewEncoder(h)
|
|
_ = enc.Encode(meta)
|
|
return fmt.Sprintf("%x", h.Sum(nil)[:16])
|
|
}
|
|
|
|
func ProvenanceJSON(m map[string]any) string {
|
|
b, err := json.Marshal(m)
|
|
if err != nil {
|
|
return "{}"
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
runes := []rune(s)
|
|
if len(runes) <= n {
|
|
return s
|
|
}
|
|
return string(runes[:n]) + "..."
|
|
}
|