akai-utils/main.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	iaSearchURL = "https://archive.org/advancedsearch.php"
	iaBaseURL   = "https://archive.org"
	userAgent   = "akai-fetch/1.0"
)

type SearchResult struct {
	Response struct {
		NumFound int            `json:"numFound"`
		Docs     []SearchDoc    `json:"docs"`
	} `json:"response"`
}

type SearchDoc struct {
	Identifier  string   `json:"identifier"`
	Title       string   `json:"title"`
	Downloads   int      `json:"downloads"`
	Format      []string `json:"format"`
}

type MetadataResult struct {
	Files []MetadataFile `json:"files"`
}

type MetadataFile struct {
	Name   string `json:"name"`
	Size   string `json:"size"`
	Format string `json:"format"`
	Source string `json:"source"`
}

func main() {
	if len(os.Args) < 2 {
		printUsage()
		os.Exit(1)
	}

	cmd := os.Args[1]
	args := os.Args[2:]

	switch cmd {
	case "search":
		cmdSearch(args)
	case "list":
		cmdList(args)
	case "download":
		cmdDownload(args)
	case "extract":
		cmdExtract(args)
	case "pipeline":
		cmdPipeline(args)
	default:
		fmt.Fprintf(os.Stderr, "unknown command: %s\n", cmd)
		printUsage()
		os.Exit(1)
	}
}

func printUsage() {
	fmt.Println(`akai-fetch — AKAI sampler ISO downloader & extractor

Usage:
  akai-fetch search [flags]             search archive.org for AKAI sampler ISOs
  akai-fetch list <identifier>          list downloadable ISO files in an item
  akai-fetch download [flags]           download ISOs (from file or by identifier)
  akai-fetch extract [flags]            extract WAVs from downloaded ISOs
  akai-fetch pipeline [flags]           full search → download → extract pipeline

Search flags:
  -query "akai sampler"    search query (default: "subject:akai AND subject:sampler")
  -filter "term"           extra term AND-ed onto the query (e.g. "vocal")
  -limit 30                max results (default: 30)
  -min-downloads 0         minimum download count filter

Download flags:
  -i "identifier"          archive.org item identifier
  -f "file.txt"            file with list of identifiers (one per line)
  -dir "./isos"            output directory (default: current dir)
  -jobs 2                  concurrent downloads (default: 2)

Extract flags:
  -dir "./isos"            directory containing ISO files
  -out "./wavs"            output directory for WAV files
  -tool "./extract_wavs.sh" path to extraction script
  -jobs 2                  concurrent extractions (default: 2)

Pipeline flags:
  -query "akai sampler"    search query (default: "subject:akai AND subject:sampler")
  -filter "term"           extra term AND-ed onto the query
  -limit 30                max results
  -dir "./output"          base output directory
  -download-jobs 2         concurrent downloads
  -extract-jobs 2          concurrent extractions`)
}

// ─── Search ────────────────────────────────────────────────────────────

func cmdSearch(args []string) {
	fs := flag.NewFlagSet("search", flag.ExitOnError)
	query := fs.String("query", "subject:akai AND subject:sampler", "search query")
	filter := fs.String("filter", "", "extra term AND-ed onto the query")
	limit := fs.Int("limit", 30, "max results")
	minDL := fs.Int("min-downloads", 0, "minimum download count")
	_ = fs.Parse(args)

	q := *query
	if *filter != "" {
		q = "(" + q + ") AND (" + *filter + ")"
	}

	results := searchArchive(q, *limit)
	sort.Slice(results, func(i, j int) bool {
		return results[i].Downloads > results[j].Downloads
	})

	fmt.Printf("Found %d items:\n\n", len(results))
	for _, r := range results {
		if *minDL > 0 && r.Downloads < *minDL {
			continue
		}
		fmt.Printf("  %-40s  downloads: %-8d  title: %s\n", r.Identifier, r.Downloads, truncate(r.Title, 50))
	}
}

func searchArchive(query string, limit int) []SearchDoc {
	params := url.Values{}
	params.Set("q", query)
	params.Set("fl[]", "identifier,title,downloads,format")
	params.Set("sort[]", "downloads desc")
	params.Set("rows", strconv.Itoa(limit))
	params.Set("page", "1")
	params.Set("output", "json")

	u := iaSearchURL + "?" + params.Encode()
	client := &http.Client{Timeout: 30 * time.Second}
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		fatal("http request: %v", err)
	}
	req.Header.Set("User-Agent", userAgent)
	resp, err := client.Do(req)
	if err != nil {
		fatal("search request: %v", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		fatal("read response: %v", err)
	}

	var sr SearchResult
	if err := json.Unmarshal(body, &sr); err != nil {
		fatal("parse response: %v", err)
	}
	return sr.Response.Docs
}

// ─── List ──────────────────────────────────────────────────────────────

func cmdList(args []string) {
	if len(args) < 1 {
		fatal("usage: akai-fetch list <identifier>")
	}
	identifier := args[0]
	files := getItemFiles(identifier)

	fmt.Printf("Files in %s:\n\n", identifier)
	for _, f := range files {
		size := f.Size
		if size == "" {
			size = "?"
		}
		fmt.Printf("  %-60s  %s\n", f.Name, size)
	}
}

func getItemFiles(identifier string) []MetadataFile {
	u := fmt.Sprintf("%s/metadata/%s", iaBaseURL, url.PathEscape(identifier))
	client := &http.Client{Timeout: 30 * time.Second}
	req, _ := http.NewRequest("GET", u, nil)
	req.Header.Set("User-Agent", userAgent)
	resp, err := client.Do(req)
	if err != nil {
		fatal("metadata request: %v", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		fatal("read metadata: %v", err)
	}

	var mr struct {
		Files []MetadataFile `json:"files"`
	}
	if err := json.Unmarshal(body, &mr); err != nil {
		fatal("parse metadata: %v", err)
	}

	var isos []MetadataFile
	for _, f := range mr.Files {
		if strings.HasSuffix(strings.ToLower(f.Name), ".iso") {
			isos = append(isos, f)
		}
	}
	return isos
}

// ─── Download ──────────────────────────────────────────────────────────

type downloadJob struct {
	Identifier string
	Name       string
	URL        string
	OutPath    string
	Size       int64
}

func cmdDownload(args []string) {
	fs := flag.NewFlagSet("download", flag.ExitOnError)
	identifier := fs.String("i", "", "archive.org item identifier")
	idFile := fs.String("f", "", "file with identifiers (one per line)")
	outDir := fs.String("dir", ".", "output directory")
	jobs := fs.Int("jobs", 2, "concurrent downloads")
	_ = fs.Parse(args)

	os.MkdirAll(*outDir, 0755)

	var identifiers []string
	if *identifier != "" {
		identifiers = append(identifiers, *identifier)
	}
	if *idFile != "" {
		data, err := os.ReadFile(*idFile)
		if err != nil {
			fatal("read id file: %v", err)
		}
		for _, line := range strings.Split(string(data), "\n") {
			line = strings.TrimSpace(line)
			if line != "" && !strings.HasPrefix(line, "#") {
				identifiers = append(identifiers, line)
			}
		}
	}

	if len(identifiers) == 0 {
		fatal("provide -i <identifier> or -f <file>")
	}

	var jobsList []downloadJob
	for _, id := range identifiers {
		files := getItemFiles(id)
		if len(files) == 0 {
			fmt.Printf("[%s] no ISO files found\n", id)
			continue
		}
		for _, f := range files {
			size, _ := strconv.ParseInt(f.Size, 10, 64)
			jobsList = append(jobsList, downloadJob{
				Identifier: id,
				Name:       f.Name,
				URL:        fmt.Sprintf("%s/download/%s/%s", iaBaseURL, url.PathEscape(id), url.PathEscape(f.Name)),
				OutPath:    filepath.Join(*outDir, f.Name),
				Size:       size,
			})
		}
	}

	if len(jobsList) == 0 {
		fmt.Println("no download jobs")
		return
	}

	fmt.Printf("Downloading %d file(s) with %d concurrent job(s)...\n\n", len(jobsList), *jobs)

	var mu sync.Mutex
	sem := make(chan struct{}, *jobs)
	var wg sync.WaitGroup
	var totalDownloaded int64

	for _, job := range jobsList {
		wg.Add(1)
		go func(j downloadJob) {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()

			size, err := downloadFile(j)
			mu.Lock()
			totalDownloaded += size
			mu.Unlock()

			if err != nil {
				fmt.Printf("  FAIL  %-40s  %v\n", j.Name, err)
			} else {
				fmt.Printf("  OK    %-40s  %s\n", j.Name, humanSize(size))
			}
		}(job)
	}
	wg.Wait()
	fmt.Printf("\nDownloaded %s total\n", humanSize(totalDownloaded))
}

func downloadFile(job downloadJob) (int64, error) {
	// Check if file already exists and has the expected size
	if fi, err := os.Stat(job.OutPath); err == nil {
		if job.Size > 0 && fi.Size() == job.Size {
			return fi.Size(), nil
		}
	}

	// Check if partial file exists for resume
	var existingSize int64
	if fi, err := os.Stat(job.OutPath); err == nil {
		existingSize = fi.Size()
	}

	client := &http.Client{Timeout: 0}
	req, err := http.NewRequest("GET", job.URL, nil)
	if err != nil {
		return 0, fmt.Errorf("request: %w", err)
	}
	req.Header.Set("User-Agent", userAgent)
	if existingSize > 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", existingSize))
	}

	resp, err := client.Do(req)
	if err != nil {
		return 0, fmt.Errorf("get: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
		return 0, fmt.Errorf("HTTP %d", resp.StatusCode)
	}

	outFile, err := os.OpenFile(job.OutPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		return 0, fmt.Errorf("open: %w", err)
	}
	defer outFile.Close()

	written, err := io.Copy(outFile, resp.Body)
	if err != nil {
		return 0, fmt.Errorf("download: %w", err)
	}
	return written + existingSize, nil
}

// ─── Extract ───────────────────────────────────────────────────────────

func cmdExtract(args []string) {
	fs := flag.NewFlagSet("extract", flag.ExitOnError)
	isoDir := fs.String("dir", ".", "directory with ISO files")
	outDir := fs.String("out", "./wavs", "output directory for WAVs")
	toolFlag := fs.String("tool", "", "path to extraction script (default: auto-detect)")
	jobs := fs.Int("jobs", 2, "concurrent extractions")
	_ = fs.Parse(args)

	toolPath := *toolFlag
	if toolPath == "" {
		toolPath = findScript("extract_wavs.sh")
	}

	entries, err := os.ReadDir(*isoDir)
	if err != nil {
		fatal("read dir: %v", err)
	}

	var isos []string
	for _, e := range entries {
		if !e.IsDir() && strings.HasSuffix(strings.ToLower(e.Name()), ".iso") {
			isos = append(isos, filepath.Join(*isoDir, e.Name()))
		}
	}

	if len(isos) == 0 {
		fmt.Println("no ISO files found in", *isoDir)
		return
	}

	fmt.Printf("Extracting %d ISO(s) with %d concurrent job(s)...\n", len(isos), *jobs)

	sem := make(chan struct{}, *jobs)
	var wg sync.WaitGroup
	var mu sync.Mutex
	results := map[string]string{}

	for _, iso := range isos {
		wg.Add(1)
		go func(isoPath string) {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()

			out := *outDir
			result, err := extractISO(toolPath, isoPath, out)
			mu.Lock()
			results[filepath.Base(isoPath)] = result
			mu.Unlock()
			if err != nil {
				fmt.Printf("  FAIL  %-40s  %v\n", filepath.Base(isoPath), err)
			} else {
				fmt.Printf("  OK    %-40s  %s\n", filepath.Base(isoPath), result)
			}
		}(iso)
	}
	wg.Wait()
}

func extractISO(toolPath, isoPath, outDir string) (string, error) {
	absTool, err := filepath.Abs(toolPath)
	if err != nil {
		return "", err
	}
	if _, err := os.Stat(absTool); os.IsNotExist(err) {
		return "", fmt.Errorf("script not found: %s", absTool)
	}

	isoOutDir := filepath.Join(outDir, sanitiseISOName(filepath.Base(isoPath)))

	if _, err := os.Stat(isoOutDir); err == nil {
		var wavCount int
		filepath.WalkDir(isoOutDir, func(p string, d os.DirEntry, err error) error {
			if err == nil && !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".wav") {
				wavCount++
			}
			return nil
		})
		if wavCount > 0 {
			return fmt.Sprintf("skipped (%d WAVs exist)", wavCount), nil
		}
	}

	cmd := exec.Command("bash", absTool, isoPath, outDir)
	cmd.Stderr = nil
	output, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("extract: %w", err)
	}

	lines := strings.Split(string(output), "\n")
	for _, line := range lines {
		if strings.Contains(line, "Total WAV files:") {
			return strings.TrimSpace(line), nil
		}
	}
	return "extracted (check output)", nil
}

func sanitiseISOName(path string) string {
	name := path
	if ext := filepath.Ext(name); ext != "" {
		name = name[:len(name)-len(ext)]
	}
	// Matches extract_wavs.sh sanitisation
	var buf strings.Builder
	for _, r := range name {
		if r == ' ' {
			buf.WriteRune('_')
		} else if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
			(r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' {
			buf.WriteRune(r)
		} else {
			buf.WriteRune('_')
		}
	}
	// Collapse multiple underscores
	s := buf.String()
	for strings.Contains(s, "__") {
		s = strings.ReplaceAll(s, "__", "_")
	}
	// Strip leading/trailing _ and .
	s = strings.Trim(s, "_.")
	return s
}

// ─── Pipeline ──────────────────────────────────────────────────────────

func cmdPipeline(args []string) {
	fs := flag.NewFlagSet("pipeline", flag.ExitOnError)
	query := fs.String("query", "subject:akai AND subject:sampler", "search query")
	filter := fs.String("filter", "", "extra term AND-ed onto the query")
	limit := fs.Int("limit", 10, "max results")
	baseDir := fs.String("dir", "./output", "base output directory")
	dlJobs := fs.Int("download-jobs", 2, "concurrent downloads")
	extJobs := fs.Int("extract-jobs", 2, "concurrent extractions")
	_ = fs.Parse(args)

	isoDir := filepath.Join(*baseDir, "isos")
	wavDir := filepath.Join(*baseDir, "wavs")
	toolPath := findScript("extract_wavs.sh")

	// Remaining args after flags are treated as explicit identifiers
	identifiers := fs.Args()

	if len(identifiers) == 0 {
		fmt.Println("=== Phase 1: Search ===")
		q := *query
		if *filter != "" {
			q = "(" + q + ") AND (" + *filter + ")"
		}
		fmt.Printf("Query: %s  Limit: %d\n\n", q, *limit)
		results := searchArchive(q, *limit)
		sort.Slice(results, func(i, j int) bool {
			return results[i].Downloads > results[j].Downloads
		})
		if len(results) == 0 {
			fmt.Println("No results found.")
			return
		}
		for _, r := range results {
			identifiers = append(identifiers, r.Identifier)
		}
		fmt.Printf("Found %d items\n\n", len(identifiers))
	} else {
		fmt.Printf("=== Processing %d specific item(s) ===\n\n", len(identifiers))
	}

	if err := os.MkdirAll(*baseDir, 0755); err != nil {
		fatal("create output dir: %v", err)
	}

	idFile := filepath.Join(*baseDir, "identifiers.txt")
	var idContent strings.Builder
	for _, id := range identifiers {
		idContent.WriteString(id + "\n")
	}
	if err := os.WriteFile(idFile, []byte(idContent.String()), 0644); err != nil {
		fatal("write ids: %v", err)
	}
	fmt.Printf("Identifiers saved to %s\n\n", idFile)

	fmt.Println("=== Phase 2: Download ===")
	cmdDownload([]string{"-f", idFile, "-dir", isoDir, "-jobs", fmt.Sprintf("%d", *dlJobs)})

	fmt.Println("\n=== Phase 3: Extract ===")
	cmdExtract([]string{"-dir", isoDir, "-out", wavDir, "-tool", toolPath, "-jobs", fmt.Sprintf("%d", *extJobs)})

	fmt.Printf("\n=== Pipeline Complete ===\n")
	fmt.Printf("  ISOs: %s\n", isoDir)
	fmt.Printf("  WAVs: %s\n", wavDir)
}

func findScript(name string) string {
	candidates := []string{
		filepath.Join(filepath.Dir(os.Args[0]), name),
		filepath.Join(filepath.Dir(os.Args[0]), "scripts", name),
		name,
		filepath.Join("scripts", name),
	}
	for _, p := range candidates {
		if _, err := os.Stat(p); err == nil {
			return p
		}
	}
	return name
}

// ─── Helpers ───────────────────────────────────────────────────────────

func fatal(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "error: "+format+"\n", args...)
	os.Exit(1)
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n-3] + "..."
}

func humanSize(bytes int64) string {
	const unit = 1024
	if bytes < unit {
		return fmt.Sprintf("%dB", bytes)
	}
	div, exp := int64(unit), 0
	for n := bytes / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}

func init() {
	// Ensure we can parse flags even with -count or other test flags
	flag.CommandLine.Parse([]string{})
}