Mainline/scripts/compare_outputs.py

#!/usr/bin/env python3
"""
Compare captured outputs from different branches or configurations.

This script loads two captured recordings and compares them frame-by-frame,
reporting any differences found.
"""

import argparse
import difflib
import json
from pathlib import Path


def load_recording(file_path: str) -> dict:
    """Load a recording from a JSON file."""
    with open(file_path, "r") as f:
        return json.load(f)


def compare_frame_buffers(buf1: list[str], buf2: list[str]) -> tuple[int, list[str]]:
    """Compare two frame buffers and return differences.

    Returns:
        tuple: (difference_count, list of difference descriptions)
    """
    differences = []

    # Check dimensions
    if len(buf1) != len(buf2):
        differences.append(f"Height mismatch: {len(buf1)} vs {len(buf2)}")

    # Check each line
    max_lines = max(len(buf1), len(buf2))
    for i in range(max_lines):
        if i >= len(buf1):
            differences.append(f"Line {i}: Missing in first buffer")
            continue
        if i >= len(buf2):
            differences.append(f"Line {i}: Missing in second buffer")
            continue

        line1 = buf1[i]
        line2 = buf2[i]

        if line1 != line2:
            # Find the specific differences in the line
            if len(line1) != len(line2):
                differences.append(
                    f"Line {i}: Length mismatch ({len(line1)} vs {len(line2)})"
                )

            # Show a snippet of the difference
            max_len = max(len(line1), len(line2))
            snippet1 = line1[:50] + "..." if len(line1) > 50 else line1
            snippet2 = line2[:50] + "..." if len(line2) > 50 else line2
            differences.append(f"Line {i}: '{snippet1}' != '{snippet2}'")

    return len(differences), differences


def compare_recordings(
    recording1: dict, recording2: dict, max_frames: int = None
) -> dict:
    """Compare two recordings frame-by-frame.

    Returns:
        dict: Comparison results with summary and detailed differences
    """
    results = {
        "summary": {},
        "frames": [],
        "total_differences": 0,
        "frames_with_differences": 0,
    }

    # Compare metadata
    results["summary"]["recording1"] = {
        "preset": recording1.get("preset", "unknown"),
        "frame_count": recording1.get("frame_count", 0),
        "width": recording1.get("width", 0),
        "height": recording1.get("height", 0),
    }
    results["summary"]["recording2"] = {
        "preset": recording2.get("preset", "unknown"),
        "frame_count": recording2.get("frame_count", 0),
        "width": recording2.get("width", 0),
        "height": recording2.get("height", 0),
    }

    # Compare frames
    frames1 = recording1.get("frames", [])
    frames2 = recording2.get("frames", [])

    num_frames = min(len(frames1), len(frames2))
    if max_frames:
        num_frames = min(num_frames, max_frames)

    print(f"Comparing {num_frames} frames...")

    for frame_idx in range(num_frames):
        frame1 = frames1[frame_idx]
        frame2 = frames2[frame_idx]

        buf1 = frame1.get("buffer", [])
        buf2 = frame2.get("buffer", [])

        diff_count, differences = compare_frame_buffers(buf1, buf2)

        if diff_count > 0:
            results["total_differences"] += diff_count
            results["frames_with_differences"] += 1
            results["frames"].append(
                {
                    "frame_number": frame_idx,
                    "differences": differences,
                    "diff_count": diff_count,
                }
            )

            if frame_idx < 5:  # Only print first 5 frames with differences
                print(f"\nFrame {frame_idx} ({diff_count} differences):")
                for diff in differences[:5]:  # Limit to 5 differences per frame
                    print(f"  - {diff}")

    # Summary
    results["summary"]["total_frames_compared"] = num_frames
    results["summary"]["frames_with_differences"] = results["frames_with_differences"]
    results["summary"]["total_differences"] = results["total_differences"]
    results["summary"]["match_percentage"] = (
        (1 - results["frames_with_differences"] / num_frames) * 100
        if num_frames > 0
        else 0
    )

    return results


def print_comparison_summary(results: dict):
    """Print a summary of the comparison results."""
    print("\n" + "=" * 80)
    print("COMPARISON SUMMARY")
    print("=" * 80)

    r1 = results["summary"]["recording1"]
    r2 = results["summary"]["recording2"]

    print(f"\nRecording 1: {r1['preset']}")
    print(
        f"  Frames: {r1['frame_count']}, Width: {r1['width']}, Height: {r1['height']}"
    )

    print(f"\nRecording 2: {r2['preset']}")
    print(
        f"  Frames: {r2['frame_count']}, Width: {r2['width']}, Height: {r2['height']}"
    )

    print(f"\nComparison:")
    print(f"  Frames compared: {results['summary']['total_frames_compared']}")
    print(f"  Frames with differences: {results['summary']['frames_with_differences']}")
    print(f"  Total differences: {results['summary']['total_differences']}")
    print(f"  Match percentage: {results['summary']['match_percentage']:.2f}%")

    if results["summary"]["match_percentage"] == 100:
        print("\n✓ Recordings match perfectly!")
    else:
        print("\n⚠ Recordings have differences.")


def main():
    parser = argparse.ArgumentParser(
        description="Compare captured outputs from different branches"
    )
    parser.add_argument(
        "recording1",
        help="First recording file (JSON)",
    )
    parser.add_argument(
        "recording2",
        help="Second recording file (JSON)",
    )
    parser.add_argument(
        "--max-frames",
        type=int,
        help="Maximum number of frames to compare",
    )
    parser.add_argument(
        "--output",
        "-o",
        help="Output file for detailed comparison results (JSON)",
    )

    args = parser.parse_args()

    # Load recordings
    print(f"Loading {args.recording1}...")
    recording1 = load_recording(args.recording1)

    print(f"Loading {args.recording2}...")
    recording2 = load_recording(args.recording2)

    # Compare
    results = compare_recordings(recording1, recording2, args.max_frames)

    # Print summary
    print_comparison_summary(results)

    # Save detailed results if requested
    if args.output:
        output_path = Path(args.output)
        output_path.parent.mkdir(parents=True, exist_ok=True)
        with open(output_path, "w") as f:
            json.dump(results, f, indent=2)
        print(f"\nDetailed results saved to {args.output}")

    return 0


if __name__ == "__main__":
    exit(main())