forked from genewildish/Mainline
feat(comparison): Add upstream vs sideline comparison framework
- Add comparison_presets.toml with 20+ preset configurations - Add comparison_capture.py for frame capture and comparison - Add run_comparison.py for running comparisons - Add test_comparison_framework.py with comprehensive tests - Add capture_upstream_comparison.py for upstream frame capture - Add tomli to dev dependencies for TOML parsing The framework supports: - Multiple preset configurations (basic, effects, camera, source, viewport) - Frame-by-frame comparison with detailed diff analysis - Performance metrics comparison - HTML report generation - Integration with sideline branch for regression testing
This commit is contained in:
144
scripts/capture_upstream_comparison.py
Normal file
144
scripts/capture_upstream_comparison.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""Capture frames from upstream Mainline for comparison testing.
|
||||
|
||||
This script should be run on the upstream/main branch to capture frames
|
||||
that will later be compared with sideline branch output.
|
||||
|
||||
Usage:
|
||||
# On upstream/main branch
|
||||
python scripts/capture_upstream_comparison.py --preset demo
|
||||
|
||||
# This will create tests/comparison_output/demo_upstream.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
|
||||
def load_preset(preset_name: str) -> dict:
|
||||
"""Load a preset from presets.toml."""
|
||||
import tomli
|
||||
|
||||
# Try user presets first
|
||||
user_presets = Path.home() / ".config" / "mainline" / "presets.toml"
|
||||
local_presets = Path("presets.toml")
|
||||
built_in_presets = Path(__file__).parent.parent / "presets.toml"
|
||||
|
||||
for preset_file in [user_presets, local_presets, built_in_presets]:
|
||||
if preset_file.exists():
|
||||
with open(preset_file, "rb") as f:
|
||||
config = tomli.load(f)
|
||||
if "presets" in config and preset_name in config["presets"]:
|
||||
return config["presets"][preset_name]
|
||||
|
||||
raise ValueError(f"Preset '{preset_name}' not found")
|
||||
|
||||
|
||||
def capture_upstream_frames(
|
||||
preset_name: str,
|
||||
frame_count: int = 30,
|
||||
output_dir: Path = Path("tests/comparison_output"),
|
||||
) -> Path:
|
||||
"""Capture frames from upstream pipeline.
|
||||
|
||||
Note: This is a simplified version that mimics upstream behavior.
|
||||
For actual upstream comparison, you may need to:
|
||||
1. Checkout upstream/main branch
|
||||
2. Run this script
|
||||
3. Copy the output file
|
||||
4. Checkout your branch
|
||||
5. Run comparison
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load preset
|
||||
preset = load_preset(preset_name)
|
||||
|
||||
# For upstream, we need to use the old monolithic rendering approach
|
||||
# This is a simplified placeholder - actual implementation depends on
|
||||
# the specific upstream architecture
|
||||
|
||||
print(f"Capturing {frame_count} frames from upstream preset '{preset_name}'")
|
||||
print("Note: This script should be run on upstream/main branch")
|
||||
print(f" for accurate comparison with sideline branch")
|
||||
|
||||
# Placeholder: In a real implementation, this would:
|
||||
# 1. Import upstream-specific modules
|
||||
# 2. Create pipeline using upstream architecture
|
||||
# 3. Capture frames
|
||||
# 4. Save to JSON
|
||||
|
||||
# For now, create a placeholder file with instructions
|
||||
placeholder_data = {
|
||||
"preset": preset_name,
|
||||
"config": preset,
|
||||
"note": "This is a placeholder file.",
|
||||
"instructions": [
|
||||
"1. Checkout upstream/main branch: git checkout main",
|
||||
"2. Run frame capture: python scripts/capture_upstream_comparison.py --preset <name>",
|
||||
"3. Copy output file to sideline branch",
|
||||
"4. Checkout sideline branch: git checkout feature/capability-based-deps",
|
||||
"5. Run comparison: python tests/run_comparison.py --preset <name>",
|
||||
],
|
||||
"frames": [], # Empty until properly captured
|
||||
}
|
||||
|
||||
output_file = output_dir / f"{preset_name}_upstream.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(placeholder_data, f, indent=2)
|
||||
|
||||
print(f"\nPlaceholder file created: {output_file}")
|
||||
print("\nTo capture actual upstream frames:")
|
||||
print("1. Ensure you are on upstream/main branch")
|
||||
print("2. This script needs to be adapted to use upstream-specific rendering")
|
||||
print("3. The captured frames will be used for comparison with sideline")
|
||||
|
||||
return output_file
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Capture frames from upstream Mainline for comparison"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preset",
|
||||
"-p",
|
||||
required=True,
|
||||
help="Preset name to capture",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--frames",
|
||||
"-f",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Number of frames to capture",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
"-o",
|
||||
type=Path,
|
||||
default=Path("tests/comparison_output"),
|
||||
help="Output directory",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
output_file = capture_upstream_frames(
|
||||
preset_name=args.preset,
|
||||
frame_count=args.frames,
|
||||
output_dir=args.output_dir,
|
||||
)
|
||||
print(f"\nCapture complete: {output_file}")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user