package main import ( "encoding/json" "flag" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "sort" "strconv" "strings" "sync" "time" ) var ( userAgent = "akai-fetch/1.0" iaSearchURL = "https://archive.org/advancedsearch.php" iaBaseURL = "https://archive.org" ) type SearchResult struct { Response struct { NumFound int `json:"numFound"` Docs []SearchDoc `json:"docs"` } `json:"response"` } type SearchDoc struct { Identifier string `json:"identifier"` Title string `json:"title"` Downloads int `json:"downloads"` Format []string `json:"format"` } type MetadataResult struct { Files []MetadataFile `json:"files"` } type MetadataFile struct { Name string `json:"name"` Size string `json:"size"` Format string `json:"format"` Source string `json:"source"` } func main() { if len(os.Args) < 2 { printUsage() os.Exit(1) } cmd := os.Args[1] args := os.Args[2:] switch cmd { case "search": cmdSearch(args) case "list": cmdList(args) case "download": cmdDownload(args) case "extract": cmdExtract(args) case "pipeline": cmdPipeline(args) case "serve": cmdServe(args) default: fmt.Fprintf(os.Stderr, "unknown command: %s\n", cmd) printUsage() os.Exit(1) } } func printUsage() { fmt.Println(`akai-fetch — AKAI sampler ISO downloader & extractor Usage: akai-fetch search [flags] search archive.org for AKAI sampler ISOs akai-fetch list list downloadable ISO files in an item akai-fetch download [flags] download ISOs (from file or by identifier) akai-fetch extract [flags] extract WAVs from downloaded ISOs akai-fetch pipeline [flags] full search → download → extract pipeline akai-fetch serve [flags] start HTTP server with web UI Search flags: -query "akai sampler" search query (default: "subject:akai AND subject:sampler") -filter "term" extra term AND-ed onto the query (e.g. "vocal") -limit 30 max results (default: 30) -min-downloads 0 minimum download count filter Download flags: -i "identifier" archive.org item identifier -f "file.txt" file with list of identifiers (one per line) -dir "./isos" output directory (default: current dir) -jobs 2 concurrent downloads (default: 2) Extract flags: -dir "./isos" directory containing ISO files -out "./wavs" output directory for WAV files -jobs 2 concurrent extractions (default: 2) Pipeline flags: -query "akai sampler" search query (default: "subject:akai AND subject:sampler") -filter "term" extra term AND-ed onto the query -limit 30 max results -dir "./output" base output directory -download-jobs 2 concurrent downloads -extract-jobs 2 concurrent extractions Serve flags: -p, --port 0 HTTP port (0 = auto-assign) --host 127.0.0.1 HTTP bind address (use 0.0.0.0 for Docker) --no-browser don't open browser automatically`) } // ─── Search ──────────────────────────────────────────────────────────── func cmdSearch(args []string) { defer func() { if e := recover(); e != nil { if fe, ok := e.(fatalErr); ok { osExit(fe.code) return } panic(e) } }() fs := flag.NewFlagSet("search", flag.ExitOnError) query := fs.String("query", "subject:akai AND subject:sampler", "search query") filter := fs.String("filter", "", "extra term AND-ed onto the query") limit := fs.Int("limit", 30, "max results") minDL := fs.Int("min-downloads", 0, "minimum download count") _ = fs.Parse(args) q := *query if *filter != "" { q = "(" + q + ") AND (" + *filter + ")" } results := searchArchive(q, *limit) sort.Slice(results, func(i, j int) bool { return results[i].Downloads > results[j].Downloads }) fmt.Printf("Found %d items:\n\n", len(results)) for _, r := range results { if *minDL > 0 && r.Downloads < *minDL { continue } fmt.Printf(" %-40s downloads: %-8d title: %s\n", r.Identifier, r.Downloads, truncate(r.Title, 50)) } } func searchArchive(query string, limit int) []SearchDoc { params := url.Values{} params.Set("q", query) params.Set("fl[]", "identifier,title,downloads,format") params.Set("sort[]", "downloads desc") params.Set("rows", strconv.Itoa(limit)) params.Set("page", "1") params.Set("output", "json") u := iaSearchURL + "?" + params.Encode() client := &http.Client{Timeout: 30 * time.Second} req, err := http.NewRequest("GET", u, nil) if err != nil { fatal("http request: %v", err) } req.Header.Set("User-Agent", userAgent) resp, err := client.Do(req) if err != nil { fatal("search request: %v", err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { fatal("read response: %v", err) } var sr SearchResult if err := json.Unmarshal(body, &sr); err != nil { fatal("parse response: %v", err) } return sr.Response.Docs } // ─── List ────────────────────────────────────────────────────────────── func cmdList(args []string) { defer func() { if e := recover(); e != nil { if fe, ok := e.(fatalErr); ok { osExit(fe.code) return } panic(e) } }() if len(args) < 1 { fatal("usage: akai-fetch list ") } identifier := args[0] files := getItemFiles(identifier) fmt.Printf("Files in %s:\n\n", identifier) for _, f := range files { size := f.Size if size == "" { size = "?" } fmt.Printf(" %-60s %s\n", f.Name, size) } } var getItemFiles = func(identifier string) []MetadataFile { u := fmt.Sprintf("%s/metadata/%s", iaBaseURL, url.PathEscape(identifier)) client := &http.Client{Timeout: 30 * time.Second} req, _ := http.NewRequest("GET", u, nil) req.Header.Set("User-Agent", userAgent) resp, err := client.Do(req) if err != nil { fatal("metadata request: %v", err) } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { fatal("read metadata: %v", err) } var mr struct { Files []MetadataFile `json:"files"` } if err := json.Unmarshal(body, &mr); err != nil { fatal("parse metadata: %v", err) } var isos []MetadataFile for _, f := range mr.Files { if strings.HasSuffix(strings.ToLower(f.Name), ".iso") { isos = append(isos, f) } } return isos } // ─── Download ────────────────────────────────────────────────────────── type downloadJob struct { Identifier string Name string URL string OutPath string Size int64 } func cmdDownload(args []string) { defer func() { if e := recover(); e != nil { if fe, ok := e.(fatalErr); ok { osExit(fe.code) return } panic(e) } }() fs := flag.NewFlagSet("download", flag.ExitOnError) identifier := fs.String("i", "", "archive.org item identifier") idFile := fs.String("f", "", "file with identifiers (one per line)") outDir := fs.String("dir", ".", "output directory") jobs := fs.Int("jobs", 2, "concurrent downloads") _ = fs.Parse(args) if err := os.MkdirAll(*outDir, 0755); err != nil { fatal("create %s: %v", *outDir, err) } var identifiers []string if *identifier != "" { identifiers = append(identifiers, *identifier) } if *idFile != "" { data, err := os.ReadFile(*idFile) if err != nil { fatal("read id file: %v", err) } for _, line := range strings.Split(string(data), "\n") { line = strings.TrimSpace(line) if line != "" && !strings.HasPrefix(line, "#") { identifiers = append(identifiers, line) } } } if len(identifiers) == 0 { fatal("provide -i or -f ") } var jobsList []downloadJob for _, id := range identifiers { files := getItemFiles(id) if len(files) == 0 { fmt.Printf("[%s] no ISO files found\n", id) continue } for _, f := range files { size, _ := strconv.ParseInt(f.Size, 10, 64) jobsList = append(jobsList, downloadJob{ Identifier: id, Name: f.Name, URL: fmt.Sprintf("%s/download/%s/%s", iaBaseURL, url.PathEscape(id), url.PathEscape(f.Name)), OutPath: filepath.Join(*outDir, f.Name), Size: size, }) } } if len(jobsList) == 0 { fmt.Println("no download jobs") return } fmt.Printf("Downloading %d file(s) with %d concurrent job(s)...\n\n", len(jobsList), *jobs) var mu sync.Mutex sem := make(chan struct{}, *jobs) var wg sync.WaitGroup var totalDownloaded int64 for _, job := range jobsList { wg.Add(1) go func(j downloadJob) { defer wg.Done() sem <- struct{}{} defer func() { <-sem }() size, err := downloadFile(j) mu.Lock() totalDownloaded += size mu.Unlock() if err != nil { fmt.Printf(" FAIL %-40s %v\n", j.Name, err) } else { fmt.Printf(" OK %-40s %s\n", j.Name, humanSize(size)) } }(job) } wg.Wait() fmt.Printf("\nDownloaded %s total\n", humanSize(totalDownloaded)) } var downloadFile = func(job downloadJob) (int64, error) { if err := os.MkdirAll(filepath.Dir(job.OutPath), 0755); err != nil { return 0, fmt.Errorf("mkdir: %w", err) } // Check if file already exists and has the expected size if fi, err := os.Stat(job.OutPath); err == nil { if job.Size > 0 && fi.Size() == job.Size { return fi.Size(), nil } } // Check if partial file exists for resume var existingSize int64 if fi, err := os.Stat(job.OutPath); err == nil { existingSize = fi.Size() } client := &http.Client{Timeout: 0} req, err := http.NewRequest("GET", job.URL, nil) if err != nil { return 0, fmt.Errorf("request: %w", err) } req.Header.Set("User-Agent", userAgent) if existingSize > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existingSize)) } resp, err := client.Do(req) if err != nil { return 0, fmt.Errorf("get: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { return 0, fmt.Errorf("HTTP %d", resp.StatusCode) } outFile, err := os.OpenFile(job.OutPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { return 0, fmt.Errorf("open: %w", err) } defer func() { _ = outFile.Close() }() written, err := io.Copy(outFile, resp.Body) if err != nil { return 0, fmt.Errorf("download: %w", err) } return written + existingSize, nil } // ─── Extract ─────────────────────────────────────────────────────────── func cmdExtract(args []string) { defer func() { if e := recover(); e != nil { if fe, ok := e.(fatalErr); ok { osExit(fe.code) return } panic(e) } }() fs := flag.NewFlagSet("extract", flag.ExitOnError) isoDir := fs.String("dir", ".", "directory with ISO files") outDir := fs.String("out", "./wavs", "output directory for WAVs") jobs := fs.Int("jobs", 2, "concurrent extractions") _ = fs.Parse(args) if err := os.MkdirAll(*outDir, 0755); err != nil { fatal("create %s: %v", *outDir, err) } entries, err := os.ReadDir(*isoDir) if err != nil { fatal("read dir: %v", err) } var isos []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".iso") { isos = append(isos, filepath.Join(*isoDir, e.Name())) } } if len(isos) == 0 { fmt.Println("no ISO files found in", *isoDir) return } fmt.Printf("Extracting %d ISO(s) with %d concurrent job(s)...\n", len(isos), *jobs) sem := make(chan struct{}, *jobs) var wg sync.WaitGroup for _, iso := range isos { wg.Add(1) go func(isoPath string) { defer wg.Done() sem <- struct{}{} defer func() { <-sem }() out := *outDir result, err := extractISO(isoPath, out) if err != nil { fmt.Printf(" FAIL %-40s %v\n", filepath.Base(isoPath), err) } else { fmt.Printf(" OK %-40s %s\n", filepath.Base(isoPath), result) } }(iso) } wg.Wait() } func sanitiseISOName(path string) string { name := path if ext := filepath.Ext(name); ext != "" { name = name[:len(name)-len(ext)] } // Matches extract_wavs.sh sanitisation var buf strings.Builder for _, r := range name { if r == ' ' { buf.WriteRune('_') } else if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' { buf.WriteRune(r) } else { buf.WriteRune('_') } } // Collapse multiple underscores s := buf.String() for strings.Contains(s, "__") { s = strings.ReplaceAll(s, "__", "_") } // Strip leading/trailing _ and . s = strings.Trim(s, "_.") return s } // ─── Pipeline ────────────────────────────────────────────────────────── func cmdPipeline(args []string) { defer func() { if e := recover(); e != nil { if fe, ok := e.(fatalErr); ok { osExit(fe.code) return } panic(e) } }() fs := flag.NewFlagSet("pipeline", flag.ExitOnError) query := fs.String("query", "subject:akai AND subject:sampler", "search query") filter := fs.String("filter", "", "extra term AND-ed onto the query") limit := fs.Int("limit", 10, "max results") baseDir := fs.String("dir", "./output", "base output directory") dlJobs := fs.Int("download-jobs", 2, "concurrent downloads") extJobs := fs.Int("extract-jobs", 2, "concurrent extractions") _ = fs.Parse(args) isoDir := filepath.Join(*baseDir, "isos") wavDir := filepath.Join(*baseDir, "wavs") // Remaining args after flags are treated as explicit identifiers identifiers := fs.Args() if len(identifiers) == 0 { fmt.Println("=== Phase 1: Search ===") q := *query if *filter != "" { q = "(" + q + ") AND (" + *filter + ")" } fmt.Printf("Query: %s Limit: %d\n\n", q, *limit) results := searchArchive(q, *limit) sort.Slice(results, func(i, j int) bool { return results[i].Downloads > results[j].Downloads }) if len(results) == 0 { fmt.Println("No results found.") return } for _, r := range results { identifiers = append(identifiers, r.Identifier) } fmt.Printf("Found %d items\n\n", len(identifiers)) } else { fmt.Printf("=== Processing %d specific item(s) ===\n\n", len(identifiers)) } if err := os.MkdirAll(*baseDir, 0755); err != nil { fatal("create output dir: %v", err) } idFile := filepath.Join(*baseDir, "identifiers.txt") var idContent strings.Builder for _, id := range identifiers { idContent.WriteString(id + "\n") } if err := os.WriteFile(idFile, []byte(idContent.String()), 0644); err != nil { fatal("write ids: %v", err) } fmt.Printf("Identifiers saved to %s\n\n", idFile) fmt.Println("=== Phase 2: Download ===") cmdDownload([]string{"-f", idFile, "-dir", isoDir, "-jobs", fmt.Sprintf("%d", *dlJobs)}) fmt.Println("\n=== Phase 3: Extract ===") cmdExtract([]string{"-dir", isoDir, "-out", wavDir, "-jobs", fmt.Sprintf("%d", *extJobs)}) fmt.Printf("\n=== Pipeline Complete ===\n") fmt.Printf(" ISOs: %s\n", isoDir) fmt.Printf(" WAVs: %s\n", wavDir) } // ─── Helpers ─────────────────────────────────────────────────────────── var osExit = os.Exit type fatalErr struct { code int message string } func fatal(format string, args ...any) { msg := fmt.Sprintf("error: "+format+"\n", args...) fmt.Fprintf(os.Stderr, msg, args...) panic(fatalErr{code: 1, message: msg}) } func truncate(s string, n int) string { if len(s) <= n { return s } if n < 3 { return s[:n] } return s[:n-3] + "..." } func humanSize(bytes int64) string { const unit = 1024 if bytes < unit { return fmt.Sprintf("%dB", bytes) } div, exp := int64(unit), 0 for n := bytes / unit; n >= unit; n /= unit { div *= unit exp++ } return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) } func init() { if err := flag.CommandLine.Parse([]string{}); err != nil { // In init context, best we can do is note the issue fmt.Fprintf(os.Stderr, "flag init: %v\n", err) } }