sideline/tests/run_comparison.py

"""Main comparison runner for upstream vs sideline testing.

This script runs comparisons between upstream and sideline implementations
using multiple presets and generates HTML reports.
"""

import argparse
import json
import sys
from pathlib import Path

from tests.comparison_capture import (
    capture_frames,
    compare_captured_outputs,
    generate_html_report,
)


def load_comparison_presets() -> list[str]:
    """Load list of comparison presets from config file.

    Returns:
        List of preset names
    """
    import tomli

    config_file = Path("tests/comparison_presets.toml")
    if not config_file.exists():
        raise FileNotFoundError(f"Comparison presets not found: {config_file}")

    with open(config_file, "rb") as f:
        config = tomli.load(f)

    presets = list(config.get("presets", {}).keys())
    # Strip "presets." prefix if present
    return [p.replace("presets.", "") for p in presets]


def run_comparison_for_preset(
    preset_name: str,
    sideline_only: bool = False,
    upstream_file: Path | None = None,
) -> dict:
    """Run comparison for a single preset.

    Args:
        preset_name: Name of preset to test
        sideline_only: If True, only capture sideline frames
        upstream_file: Path to upstream captured output (if not None, use this instead of capturing)

    Returns:
        Comparison result dict
    """
    print(f"  Running preset: {preset_name}")

    # Capture sideline frames
    sideline_data = capture_frames(preset_name, frame_count=30)
    sideline_file = Path(f"tests/comparison_output/{preset_name}_sideline.json")

    if sideline_only:
        return {
            "preset": preset_name,
            "status": "sideline_only",
            "sideline_file": str(sideline_file),
        }

    # Use provided upstream file or look for it
    if upstream_file:
        upstream_path = upstream_file
    else:
        upstream_path = Path(f"tests/comparison_output/{preset_name}_upstream.json")

    if not upstream_path.exists():
        print(f"    Warning: Upstream file not found: {upstream_path}")
        return {
            "preset": preset_name,
            "status": "missing_upstream",
            "sideline_file": str(sideline_file),
            "upstream_file": str(upstream_path),
        }

    # Compare outputs
    try:
        comparison_result = compare_captured_outputs(
            sideline_file=sideline_file,
            upstream_file=upstream_path,
        )
        comparison_result["status"] = "success"
        return comparison_result
    except Exception as e:
        print(f"    Error comparing outputs: {e}")
        return {
            "preset": preset_name,
            "status": "error",
            "error": str(e),
            "sideline_file": str(sideline_file),
            "upstream_file": str(upstream_path),
        }


def main():
    """Main entry point for comparison runner."""
    parser = argparse.ArgumentParser(
        description="Run comparison tests between upstream and sideline implementations"
    )
    parser.add_argument(
        "--preset",
        "-p",
        help="Run specific preset (can be specified multiple times)",
        action="append",
        dest="presets",
    )
    parser.add_argument(
        "--all",
        "-a",
        help="Run all comparison presets",
        action="store_true",
    )
    parser.add_argument(
        "--sideline-only",
        "-s",
        help="Only capture sideline frames (no comparison)",
        action="store_true",
    )
    parser.add_argument(
        "--upstream-file",
        "-u",
        help="Path to upstream captured output file",
        type=Path,
    )
    parser.add_argument(
        "--output-dir",
        "-o",
        help="Output directory for captured frames and reports",
        type=Path,
        default=Path("tests/comparison_output"),
    )
    parser.add_argument(
        "--no-report",
        help="Skip HTML report generation",
        action="store_true",
    )

    args = parser.parse_args()

    # Determine which presets to run
    if args.presets:
        presets_to_run = args.presets
    elif args.all:
        presets_to_run = load_comparison_presets()
    else:
        print("Error: Either --preset or --all must be specified")
        print(f"Available presets: {', '.join(load_comparison_presets())}")
        sys.exit(1)

    print(f"Running comparison for {len(presets_to_run)} preset(s)")
    print(f"Output directory: {args.output_dir}")
    print()

    # Run comparisons
    results = []
    for preset_name in presets_to_run:
        try:
            result = run_comparison_for_preset(
                preset_name,
                sideline_only=args.sideline_only,
                upstream_file=args.upstream_file,
            )
            results.append(result)

            if result["status"] == "success":
                match_pct = result["stats"]["match_percentage"]
                print(f"    ✓ Match: {match_pct:.1f}%")
            elif result["status"] == "missing_upstream":
                print(f"    ⚠ Missing upstream file")
            elif result["status"] == "error":
                print(f"    ✗ Error: {result['error']}")
            else:
                print(f"    ✓ Captured sideline only")

        except Exception as e:
            print(f"    ✗ Failed: {e}")
            results.append(
                {
                    "preset": preset_name,
                    "status": "failed",
                    "error": str(e),
                }
            )

    # Generate HTML report
    if not args.no_report and not args.sideline_only:
        successful_results = [r for r in results if r.get("status") == "success"]
        if successful_results:
            print(f"\nGenerating HTML report...")
            report_file = generate_html_report(successful_results, args.output_dir)
            print(f"  Report saved to: {report_file}")

            # Also save summary JSON
            summary_file = args.output_dir / "comparison_summary.json"
            with open(summary_file, "w") as f:
                json.dump(
                    {
                        "timestamp": __import__("datetime").datetime.now().isoformat(),
                        "presets_tested": [r["preset"] for r in results],
                        "results": results,
                    },
                    f,
                    indent=2,
                )
            print(f"  Summary saved to: {summary_file}")
        else:
            print(f"\nNote: No successful comparisons to report.")
            print(f"      Capture files saved in {args.output_dir}")
            print(f"      Run comparison when upstream files are available.")

    # Print summary
    print("\n" + "=" * 60)
    print("SUMMARY")
    print("=" * 60)

    status_counts = {}
    for result in results:
        status = result.get("status", "unknown")
        status_counts[status] = status_counts.get(status, 0) + 1

    for status, count in sorted(status_counts.items()):
        print(f"  {status}: {count}")

    if "success" in status_counts:
        successful_results = [r for r in results if r.get("status") == "success"]
        avg_match = sum(
            r["stats"]["match_percentage"] for r in successful_results
        ) / len(successful_results)
        print(f"\n  Average match rate: {avg_match:.1f}%")

    # Exit with error code if any failures
    if any(r.get("status") in ["error", "failed"] for r in results):
        sys.exit(1)


if __name__ == "__main__":
    main()