forked from genewildish/Mainline
feat(comparison): Add upstream vs sideline comparison framework
- Add comparison_presets.toml with 20+ preset configurations - Add comparison_capture.py for frame capture and comparison - Add run_comparison.py for running comparisons - Add test_comparison_framework.py with comprehensive tests - Add capture_upstream_comparison.py for upstream frame capture - Add tomli to dev dependencies for TOML parsing The framework supports: - Multiple preset configurations (basic, effects, camera, source, viewport) - Frame-by-frame comparison with detailed diff analysis - Performance metrics comparison - HTML report generation - Integration with sideline branch for regression testing
This commit is contained in:
502
tests/comparison_capture.py
Normal file
502
tests/comparison_capture.py
Normal file
@@ -0,0 +1,502 @@
|
||||
"""Frame capture utilities for upstream vs sideline comparison.
|
||||
|
||||
This module provides functions to capture frames from both upstream and sideline
|
||||
implementations for visual comparison and performance analysis.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import tomli
|
||||
|
||||
from engine.pipeline import Pipeline, PipelineConfig, PipelineContext
|
||||
from engine.pipeline.params import PipelineParams
|
||||
|
||||
|
||||
def load_comparison_preset(preset_name: str) -> Any:
|
||||
"""Load a comparison preset from comparison_presets.toml.
|
||||
|
||||
Args:
|
||||
preset_name: Name of the preset to load
|
||||
|
||||
Returns:
|
||||
Preset configuration dictionary
|
||||
"""
|
||||
presets_file = Path("tests/comparison_presets.toml")
|
||||
if not presets_file.exists():
|
||||
raise FileNotFoundError(f"Comparison presets file not found: {presets_file}")
|
||||
|
||||
with open(presets_file, "rb") as f:
|
||||
config = tomli.load(f)
|
||||
|
||||
presets = config.get("presets", {})
|
||||
full_name = (
|
||||
f"presets.{preset_name}"
|
||||
if not preset_name.startswith("presets.")
|
||||
else preset_name
|
||||
)
|
||||
simple_name = (
|
||||
preset_name.replace("presets.", "")
|
||||
if preset_name.startswith("presets.")
|
||||
else preset_name
|
||||
)
|
||||
|
||||
if full_name in presets:
|
||||
return presets[full_name]
|
||||
elif simple_name in presets:
|
||||
return presets[simple_name]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Preset '{preset_name}' not found in {presets_file}. Available: {list(presets.keys())}"
|
||||
)
|
||||
|
||||
|
||||
def capture_frames(
|
||||
preset_name: str,
|
||||
frame_count: int = 30,
|
||||
output_dir: Path = Path("tests/comparison_output"),
|
||||
) -> Dict[str, Any]:
|
||||
"""Capture frames from sideline pipeline using a preset.
|
||||
|
||||
Args:
|
||||
preset_name: Name of preset to use
|
||||
frame_count: Number of frames to capture
|
||||
output_dir: Directory to save captured frames
|
||||
|
||||
Returns:
|
||||
Dictionary with captured frames and metadata
|
||||
"""
|
||||
from engine.pipeline.presets import get_preset
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load preset - try comparison presets first, then built-in presets
|
||||
try:
|
||||
preset = load_comparison_preset(preset_name)
|
||||
# Convert dict to object-like access
|
||||
from types import SimpleNamespace
|
||||
|
||||
preset = SimpleNamespace(**preset)
|
||||
except (FileNotFoundError, ValueError):
|
||||
# Fall back to built-in presets
|
||||
preset = get_preset(preset_name)
|
||||
if not preset:
|
||||
raise ValueError(
|
||||
f"Preset '{preset_name}' not found in comparison or built-in presets"
|
||||
)
|
||||
|
||||
# Create pipeline config from preset
|
||||
config = PipelineConfig(
|
||||
source=preset.source,
|
||||
display="null", # Always use null display for capture
|
||||
camera=preset.camera,
|
||||
effects=preset.effects,
|
||||
)
|
||||
|
||||
# Create pipeline
|
||||
ctx = PipelineContext()
|
||||
ctx.terminal_width = preset.viewport_width
|
||||
ctx.terminal_height = preset.viewport_height
|
||||
pipeline = Pipeline(config=config, context=ctx)
|
||||
|
||||
# Create params
|
||||
params = PipelineParams(
|
||||
viewport_width=preset.viewport_width,
|
||||
viewport_height=preset.viewport_height,
|
||||
)
|
||||
ctx.params = params
|
||||
|
||||
# Add message overlay stage if enabled
|
||||
if getattr(preset, "enable_message_overlay", False):
|
||||
from engine.pipeline.adapters import MessageOverlayConfig, MessageOverlayStage
|
||||
|
||||
overlay_config = MessageOverlayConfig(
|
||||
enabled=True,
|
||||
display_secs=30,
|
||||
)
|
||||
pipeline.add_stage(
|
||||
"message_overlay", MessageOverlayStage(config=overlay_config)
|
||||
)
|
||||
|
||||
# Build pipeline
|
||||
pipeline.build()
|
||||
|
||||
# Capture frames
|
||||
frames = []
|
||||
start_time = time.time()
|
||||
|
||||
for i in range(frame_count):
|
||||
frame_start = time.time()
|
||||
stage_result = pipeline.execute()
|
||||
frame_time = time.time() - frame_start
|
||||
|
||||
# Extract buffer from result
|
||||
buffer = stage_result.data if stage_result.success else []
|
||||
|
||||
frames.append(
|
||||
{
|
||||
"frame_number": i,
|
||||
"buffer": buffer,
|
||||
"width": preset.viewport_width,
|
||||
"height": preset.viewport_height,
|
||||
"render_time_ms": frame_time * 1000,
|
||||
}
|
||||
)
|
||||
|
||||
total_time = time.time() - start_time
|
||||
|
||||
# Save captured data
|
||||
output_file = output_dir / f"{preset_name}_sideline.json"
|
||||
captured_data = {
|
||||
"preset": preset_name,
|
||||
"config": {
|
||||
"source": preset.source,
|
||||
"camera": preset.camera,
|
||||
"effects": preset.effects,
|
||||
"viewport_width": preset.viewport_width,
|
||||
"viewport_height": preset.viewport_height,
|
||||
"enable_message_overlay": getattr(preset, "enable_message_overlay", False),
|
||||
},
|
||||
"capture_stats": {
|
||||
"frame_count": frame_count,
|
||||
"total_time_ms": total_time * 1000,
|
||||
"avg_frame_time_ms": (total_time * 1000) / frame_count,
|
||||
"fps": frame_count / total_time if total_time > 0 else 0,
|
||||
},
|
||||
"frames": frames,
|
||||
}
|
||||
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(captured_data, f, indent=2)
|
||||
|
||||
return captured_data
|
||||
|
||||
|
||||
def compare_captured_outputs(
|
||||
sideline_file: Path,
|
||||
upstream_file: Path,
|
||||
output_dir: Path = Path("tests/comparison_output"),
|
||||
) -> Dict[str, Any]:
|
||||
"""Compare captured outputs from sideline and upstream.
|
||||
|
||||
Args:
|
||||
sideline_file: Path to sideline captured output
|
||||
upstream_file: Path to upstream captured output
|
||||
output_dir: Directory to save comparison results
|
||||
|
||||
Returns:
|
||||
Dictionary with comparison results
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load captured data
|
||||
with open(sideline_file) as f:
|
||||
sideline_data = json.load(f)
|
||||
|
||||
with open(upstream_file) as f:
|
||||
upstream_data = json.load(f)
|
||||
|
||||
# Compare configurations
|
||||
config_diff = {}
|
||||
for key in [
|
||||
"source",
|
||||
"camera",
|
||||
"effects",
|
||||
"viewport_width",
|
||||
"viewport_height",
|
||||
"enable_message_overlay",
|
||||
]:
|
||||
sideline_val = sideline_data["config"].get(key)
|
||||
upstream_val = upstream_data["config"].get(key)
|
||||
if sideline_val != upstream_val:
|
||||
config_diff[key] = {"sideline": sideline_val, "upstream": upstream_val}
|
||||
|
||||
# Compare frame counts
|
||||
sideline_frames = len(sideline_data["frames"])
|
||||
upstream_frames = len(upstream_data["frames"])
|
||||
frame_count_match = sideline_frames == upstream_frames
|
||||
|
||||
# Compare individual frames
|
||||
frame_comparisons = []
|
||||
total_diff = 0
|
||||
max_diff = 0
|
||||
identical_frames = 0
|
||||
|
||||
min_frames = min(sideline_frames, upstream_frames)
|
||||
for i in range(min_frames):
|
||||
sideline_frame = sideline_data["frames"][i]
|
||||
upstream_frame = upstream_data["frames"][i]
|
||||
|
||||
sideline_buffer = sideline_frame["buffer"]
|
||||
upstream_buffer = upstream_frame["buffer"]
|
||||
|
||||
# Compare buffers line by line
|
||||
line_diffs = []
|
||||
frame_diff = 0
|
||||
max_lines = max(len(sideline_buffer), len(upstream_buffer))
|
||||
|
||||
for line_idx in range(max_lines):
|
||||
sideline_line = (
|
||||
sideline_buffer[line_idx] if line_idx < len(sideline_buffer) else ""
|
||||
)
|
||||
upstream_line = (
|
||||
upstream_buffer[line_idx] if line_idx < len(upstream_buffer) else ""
|
||||
)
|
||||
|
||||
if sideline_line != upstream_line:
|
||||
line_diffs.append(
|
||||
{
|
||||
"line": line_idx,
|
||||
"sideline": sideline_line,
|
||||
"upstream": upstream_line,
|
||||
}
|
||||
)
|
||||
frame_diff += 1
|
||||
|
||||
if frame_diff == 0:
|
||||
identical_frames += 1
|
||||
|
||||
total_diff += frame_diff
|
||||
max_diff = max(max_diff, frame_diff)
|
||||
|
||||
frame_comparisons.append(
|
||||
{
|
||||
"frame_number": i,
|
||||
"differences": frame_diff,
|
||||
"line_diffs": line_diffs[
|
||||
:5
|
||||
], # Only store first 5 differences per frame
|
||||
"render_time_diff_ms": sideline_frame.get("render_time_ms", 0)
|
||||
- upstream_frame.get("render_time_ms", 0),
|
||||
}
|
||||
)
|
||||
|
||||
# Calculate statistics
|
||||
stats = {
|
||||
"total_frames_compared": min_frames,
|
||||
"identical_frames": identical_frames,
|
||||
"frames_with_differences": min_frames - identical_frames,
|
||||
"total_differences": total_diff,
|
||||
"max_differences_per_frame": max_diff,
|
||||
"avg_differences_per_frame": total_diff / min_frames if min_frames > 0 else 0,
|
||||
"match_percentage": (identical_frames / min_frames * 100)
|
||||
if min_frames > 0
|
||||
else 0,
|
||||
}
|
||||
|
||||
# Compare performance stats
|
||||
sideline_stats = sideline_data.get("capture_stats", {})
|
||||
upstream_stats = upstream_data.get("capture_stats", {})
|
||||
performance_comparison = {
|
||||
"sideline": {
|
||||
"total_time_ms": sideline_stats.get("total_time_ms", 0),
|
||||
"avg_frame_time_ms": sideline_stats.get("avg_frame_time_ms", 0),
|
||||
"fps": sideline_stats.get("fps", 0),
|
||||
},
|
||||
"upstream": {
|
||||
"total_time_ms": upstream_stats.get("total_time_ms", 0),
|
||||
"avg_frame_time_ms": upstream_stats.get("avg_frame_time_ms", 0),
|
||||
"fps": upstream_stats.get("fps", 0),
|
||||
},
|
||||
"diff": {
|
||||
"total_time_ms": sideline_stats.get("total_time_ms", 0)
|
||||
- upstream_stats.get("total_time_ms", 0),
|
||||
"avg_frame_time_ms": sideline_stats.get("avg_frame_time_ms", 0)
|
||||
- upstream_stats.get("avg_frame_time_ms", 0),
|
||||
"fps": sideline_stats.get("fps", 0) - upstream_stats.get("fps", 0),
|
||||
},
|
||||
}
|
||||
|
||||
# Build comparison result
|
||||
result = {
|
||||
"preset": sideline_data["preset"],
|
||||
"config_diff": config_diff,
|
||||
"frame_count_match": frame_count_match,
|
||||
"stats": stats,
|
||||
"performance_comparison": performance_comparison,
|
||||
"frame_comparisons": frame_comparisons,
|
||||
"sideline_file": str(sideline_file),
|
||||
"upstream_file": str(upstream_file),
|
||||
}
|
||||
|
||||
# Save comparison result
|
||||
output_file = output_dir / f"{sideline_data['preset']}_comparison.json"
|
||||
with open(output_file, "w") as f:
|
||||
json.dump(result, f, indent=2)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_html_report(
|
||||
comparison_results: List[Dict[str, Any]],
|
||||
output_dir: Path = Path("tests/comparison_output"),
|
||||
) -> Path:
|
||||
"""Generate HTML report from comparison results.
|
||||
|
||||
Args:
|
||||
comparison_results: List of comparison results
|
||||
output_dir: Directory to save HTML report
|
||||
|
||||
Returns:
|
||||
Path to generated HTML report
|
||||
"""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
html_content = """
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Mainline Comparison Report</title>
|
||||
<style>
|
||||
body { font-family: Arial, sans-serif; margin: 20px; background: #f5f5f5; }
|
||||
.header { background: #2c3e50; color: white; padding: 20px; border-radius: 5px; }
|
||||
.summary { background: white; padding: 15px; margin: 10px 0; border-radius: 5px; }
|
||||
.preset { background: white; margin: 10px 0; padding: 15px; border-radius: 5px; }
|
||||
.preset-header { font-size: 1.2em; font-weight: bold; margin-bottom: 10px; }
|
||||
.stats { display: grid; grid-template-columns: repeat(3, 1fr); gap: 10px; margin: 10px 0; }
|
||||
.stat-box { background: #ecf0f1; padding: 10px; border-radius: 3px; text-align: center; }
|
||||
.stat-value { font-size: 1.5em; font-weight: bold; }
|
||||
.stat-label { font-size: 0.9em; color: #7f8c8d; }
|
||||
.match { color: #27ae60; }
|
||||
.mismatch { color: #e74c3c; }
|
||||
.warning { color: #f39c12; }
|
||||
.frame-comparison { margin: 10px 0; }
|
||||
.frame-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
|
||||
.frame-box { background: #ecf0f1; padding: 10px; border-radius: 3px; }
|
||||
.frame-header { font-weight: bold; margin-bottom: 5px; }
|
||||
.diff-line { background: #ffeaa7; padding: 2px 5px; margin: 2px 0; font-family: monospace; font-size: 0.8em; }
|
||||
.performance { background: #e8f4f8; padding: 15px; margin: 10px 0; border-radius: 5px; }
|
||||
.performance-grid { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 10px; }
|
||||
.perf-box { text-align: center; padding: 10px; }
|
||||
.perf-sideline { background: #d5f4e6; }
|
||||
.perf-upstream { background: #fde8e8; }
|
||||
.perf-diff { background: #fff3cd; }
|
||||
.timestamp { color: #7f8c8d; font-size: 0.9em; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Mainline Pipeline Comparison Report</h1>
|
||||
<p class="timestamp">Generated: {{timestamp}}</p>
|
||||
</div>
|
||||
|
||||
<div class="summary">
|
||||
<h2>Summary</h2>
|
||||
<div class="stats">
|
||||
<div class="stat-box">
|
||||
<div class="stat-value" id="total-presets">0</div>
|
||||
<div class="stat-label">Presets Tested</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="stat-value" id="total-match">0%</div>
|
||||
<div class="stat-label">Average Match Rate</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="stat-value" id="total-frames">0</div>
|
||||
<div class="stat-label">Total Frames Compared</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="preset-results">
|
||||
<!-- Preset results will be inserted here -->
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const comparisonData = {{comparison_data}};
|
||||
const summary = {{summary}};
|
||||
|
||||
// Update summary
|
||||
document.getElementById('total-presets').textContent = summary.total_presets;
|
||||
document.getElementById('total-match').textContent = summary.average_match.toFixed(1) + '%';
|
||||
document.getElementById('total-frames').textContent = summary.total_frames;
|
||||
|
||||
// Generate preset results
|
||||
const resultsContainer = document.getElementById('preset-results');
|
||||
|
||||
comparisonData.forEach(result => {
|
||||
const presetDiv = document.createElement('div');
|
||||
presetDiv.className = 'preset';
|
||||
|
||||
const matchClass = result.stats.match_percentage >= 95 ? 'match' :
|
||||
result.stats.match_percentage >= 80 ? 'warning' : 'mismatch';
|
||||
|
||||
presetDiv.innerHTML = `
|
||||
<div class="preset-header">${result.preset}</div>
|
||||
<div class="stats">
|
||||
<div class="stat-box">
|
||||
<div class="stat-value ${matchClass}">${result.stats.match_percentage.toFixed(1)}%</div>
|
||||
<div class="stat-label">Frame Match Rate</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="stat-value">${result.stats.total_frames_compared}</div>
|
||||
<div class="stat-label">Frames Compared</div>
|
||||
</div>
|
||||
<div class="stat-box">
|
||||
<div class="stat-value">${result.stats.identical_frames}</div>
|
||||
<div class="stat-label">Identical Frames</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="performance">
|
||||
<h3>Performance Comparison</h3>
|
||||
<div class="performance-grid">
|
||||
<div class="perf-box perf-sideline">
|
||||
<div>Sideline</div>
|
||||
<div class="stat-value">${result.performance_comparison.sideline.avg_frame_time_ms.toFixed(2)}ms</div>
|
||||
<div class="stat-label">${result.performance_comparison.sideline.fps.toFixed(1)} FPS</div>
|
||||
</div>
|
||||
<div class="perf-box perf-upstream">
|
||||
<div>Upstream</div>
|
||||
<div class="stat-value">${result.performance_comparison.upstream.avg_frame_time_ms.toFixed(2)}ms</div>
|
||||
<div class="stat-label">${result.performance_comparison.upstream.fps.toFixed(1)} FPS</div>
|
||||
</div>
|
||||
<div class="perf-box perf-diff">
|
||||
<div>Difference</div>
|
||||
<div class="stat-value">${result.performance_comparison.diff.avg_frame_time_ms.toFixed(2)}ms</div>
|
||||
<div class="stat-label">${result.performance_comparison.diff.fps.toFixed(1)} FPS</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
resultsContainer.appendChild(presetDiv);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Generate comparison data for JavaScript
|
||||
comparison_data_json = json.dumps(comparison_results)
|
||||
|
||||
# Calculate summary statistics
|
||||
total_presets = len(comparison_results)
|
||||
total_frames = sum(r["stats"]["total_frames_compared"] for r in comparison_results)
|
||||
total_identical = sum(r["stats"]["identical_frames"] for r in comparison_results)
|
||||
average_match = (total_identical / total_frames * 100) if total_frames > 0 else 0
|
||||
|
||||
summary = {
|
||||
"total_presets": total_presets,
|
||||
"total_frames": total_frames,
|
||||
"total_identical": total_identical,
|
||||
"average_match": average_match,
|
||||
}
|
||||
|
||||
# Replace placeholders
|
||||
html_content = html_content.replace(
|
||||
"{{timestamp}}", time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
)
|
||||
html_content = html_content.replace("{{comparison_data}}", comparison_data_json)
|
||||
html_content = html_content.replace("{{summary}}", json.dumps(summary))
|
||||
|
||||
# Save HTML report
|
||||
output_file = output_dir / "comparison_report.html"
|
||||
with open(output_file, "w") as f:
|
||||
f.write(html_content)
|
||||
|
||||
return output_file
|
||||
253
tests/comparison_presets.toml
Normal file
253
tests/comparison_presets.toml
Normal file
@@ -0,0 +1,253 @@
|
||||
# Comparison Presets for Upstream vs Sideline Testing
|
||||
# These presets are designed to test various pipeline configurations
|
||||
# to ensure visual equivalence and performance parity
|
||||
|
||||
# ============================================
|
||||
# CORE PIPELINE TESTS (Basic functionality)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-basic]
|
||||
description = "Comparison: Basic pipeline, no effects"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-with-message-overlay]
|
||||
description = "Comparison: Basic pipeline with message overlay"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = true
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# EFFECT TESTS (Various effect combinations)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-single-effect]
|
||||
description = "Comparison: Single effect (border)"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = ["border"]
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-multiple-effects]
|
||||
description = "Comparison: Multiple effects chain"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = ["border", "tint", "hud"]
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-all-effects]
|
||||
description = "Comparison: All available effects"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = ["border", "tint", "hud", "fade", "noise", "glitch"]
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# CAMERA MODE TESTS (Different viewport behaviors)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-camera-feed]
|
||||
description = "Comparison: Feed camera mode"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-camera-scroll]
|
||||
description = "Comparison: Scroll camera mode"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "scroll"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
camera_speed = 0.5
|
||||
|
||||
[presets.comparison-camera-horizontal]
|
||||
description = "Comparison: Horizontal camera mode"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "horizontal"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# SOURCE TESTS (Different data sources)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-source-headlines]
|
||||
description = "Comparison: Headlines source"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-source-poetry]
|
||||
description = "Comparison: Poetry source"
|
||||
source = "poetry"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-source-empty]
|
||||
description = "Comparison: Empty source (blank canvas)"
|
||||
source = "empty"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# DIMENSION TESTS (Different viewport sizes)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-small-viewport]
|
||||
description = "Comparison: Small viewport"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 60
|
||||
viewport_height = 20
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-large-viewport]
|
||||
description = "Comparison: Large viewport"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 120
|
||||
viewport_height = 40
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-wide-viewport]
|
||||
description = "Comparison: Wide viewport"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 160
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# COMPREHENSIVE TESTS (Combined scenarios)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-comprehensive-1]
|
||||
description = "Comparison: Headlines + Effects + Message Overlay"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = ["border", "tint"]
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = true
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-comprehensive-2]
|
||||
description = "Comparison: Poetry + Camera Scroll + Effects"
|
||||
source = "poetry"
|
||||
display = "null"
|
||||
camera = "scroll"
|
||||
effects = ["fade", "noise"]
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
camera_speed = 0.3
|
||||
|
||||
[presets.comparison-comprehensive-3]
|
||||
description = "Comparison: Headlines + Horizontal Camera + All Effects"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "horizontal"
|
||||
effects = ["border", "tint", "hud", "fade"]
|
||||
viewport_width = 100
|
||||
viewport_height = 30
|
||||
enable_message_overlay = true
|
||||
frame_count = 30
|
||||
|
||||
# ============================================
|
||||
# REGRESSION TESTS (Specific edge cases)
|
||||
# ============================================
|
||||
|
||||
[presets.comparison-regression-empty-message]
|
||||
description = "Regression: Empty message overlay"
|
||||
source = "empty"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 24
|
||||
enable_message_overlay = true
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-regression-narrow-viewport]
|
||||
description = "Regression: Very narrow viewport with long text"
|
||||
source = "headlines"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 40
|
||||
viewport_height = 24
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
|
||||
[presets.comparison-regression-tall-viewport]
|
||||
description = "Regression: Tall viewport with few items"
|
||||
source = "empty"
|
||||
display = "null"
|
||||
camera = "feed"
|
||||
effects = []
|
||||
viewport_width = 80
|
||||
viewport_height = 60
|
||||
enable_message_overlay = false
|
||||
frame_count = 30
|
||||
243
tests/run_comparison.py
Normal file
243
tests/run_comparison.py
Normal file
@@ -0,0 +1,243 @@
|
||||
"""Main comparison runner for upstream vs sideline testing.
|
||||
|
||||
This script runs comparisons between upstream and sideline implementations
|
||||
using multiple presets and generates HTML reports.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from tests.comparison_capture import (
|
||||
capture_frames,
|
||||
compare_captured_outputs,
|
||||
generate_html_report,
|
||||
)
|
||||
|
||||
|
||||
def load_comparison_presets() -> list[str]:
|
||||
"""Load list of comparison presets from config file.
|
||||
|
||||
Returns:
|
||||
List of preset names
|
||||
"""
|
||||
import tomli
|
||||
|
||||
config_file = Path("tests/comparison_presets.toml")
|
||||
if not config_file.exists():
|
||||
raise FileNotFoundError(f"Comparison presets not found: {config_file}")
|
||||
|
||||
with open(config_file, "rb") as f:
|
||||
config = tomli.load(f)
|
||||
|
||||
presets = list(config.get("presets", {}).keys())
|
||||
# Strip "presets." prefix if present
|
||||
return [p.replace("presets.", "") for p in presets]
|
||||
|
||||
|
||||
def run_comparison_for_preset(
|
||||
preset_name: str,
|
||||
sideline_only: bool = False,
|
||||
upstream_file: Path | None = None,
|
||||
) -> dict:
|
||||
"""Run comparison for a single preset.
|
||||
|
||||
Args:
|
||||
preset_name: Name of preset to test
|
||||
sideline_only: If True, only capture sideline frames
|
||||
upstream_file: Path to upstream captured output (if not None, use this instead of capturing)
|
||||
|
||||
Returns:
|
||||
Comparison result dict
|
||||
"""
|
||||
print(f" Running preset: {preset_name}")
|
||||
|
||||
# Capture sideline frames
|
||||
sideline_data = capture_frames(preset_name, frame_count=30)
|
||||
sideline_file = Path(f"tests/comparison_output/{preset_name}_sideline.json")
|
||||
|
||||
if sideline_only:
|
||||
return {
|
||||
"preset": preset_name,
|
||||
"status": "sideline_only",
|
||||
"sideline_file": str(sideline_file),
|
||||
}
|
||||
|
||||
# Use provided upstream file or look for it
|
||||
if upstream_file:
|
||||
upstream_path = upstream_file
|
||||
else:
|
||||
upstream_path = Path(f"tests/comparison_output/{preset_name}_upstream.json")
|
||||
|
||||
if not upstream_path.exists():
|
||||
print(f" Warning: Upstream file not found: {upstream_path}")
|
||||
return {
|
||||
"preset": preset_name,
|
||||
"status": "missing_upstream",
|
||||
"sideline_file": str(sideline_file),
|
||||
"upstream_file": str(upstream_path),
|
||||
}
|
||||
|
||||
# Compare outputs
|
||||
try:
|
||||
comparison_result = compare_captured_outputs(
|
||||
sideline_file=sideline_file,
|
||||
upstream_file=upstream_path,
|
||||
)
|
||||
comparison_result["status"] = "success"
|
||||
return comparison_result
|
||||
except Exception as e:
|
||||
print(f" Error comparing outputs: {e}")
|
||||
return {
|
||||
"preset": preset_name,
|
||||
"status": "error",
|
||||
"error": str(e),
|
||||
"sideline_file": str(sideline_file),
|
||||
"upstream_file": str(upstream_path),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for comparison runner."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run comparison tests between upstream and sideline implementations"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preset",
|
||||
"-p",
|
||||
help="Run specific preset (can be specified multiple times)",
|
||||
action="append",
|
||||
dest="presets",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
"-a",
|
||||
help="Run all comparison presets",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sideline-only",
|
||||
"-s",
|
||||
help="Only capture sideline frames (no comparison)",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--upstream-file",
|
||||
"-u",
|
||||
help="Path to upstream captured output file",
|
||||
type=Path,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
"-o",
|
||||
help="Output directory for captured frames and reports",
|
||||
type=Path,
|
||||
default=Path("tests/comparison_output"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-report",
|
||||
help="Skip HTML report generation",
|
||||
action="store_true",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Determine which presets to run
|
||||
if args.presets:
|
||||
presets_to_run = args.presets
|
||||
elif args.all:
|
||||
presets_to_run = load_comparison_presets()
|
||||
else:
|
||||
print("Error: Either --preset or --all must be specified")
|
||||
print(f"Available presets: {', '.join(load_comparison_presets())}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Running comparison for {len(presets_to_run)} preset(s)")
|
||||
print(f"Output directory: {args.output_dir}")
|
||||
print()
|
||||
|
||||
# Run comparisons
|
||||
results = []
|
||||
for preset_name in presets_to_run:
|
||||
try:
|
||||
result = run_comparison_for_preset(
|
||||
preset_name,
|
||||
sideline_only=args.sideline_only,
|
||||
upstream_file=args.upstream_file,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
if result["status"] == "success":
|
||||
match_pct = result["stats"]["match_percentage"]
|
||||
print(f" ✓ Match: {match_pct:.1f}%")
|
||||
elif result["status"] == "missing_upstream":
|
||||
print(f" ⚠ Missing upstream file")
|
||||
elif result["status"] == "error":
|
||||
print(f" ✗ Error: {result['error']}")
|
||||
else:
|
||||
print(f" ✓ Captured sideline only")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed: {e}")
|
||||
results.append(
|
||||
{
|
||||
"preset": preset_name,
|
||||
"status": "failed",
|
||||
"error": str(e),
|
||||
}
|
||||
)
|
||||
|
||||
# Generate HTML report
|
||||
if not args.no_report and not args.sideline_only:
|
||||
successful_results = [r for r in results if r.get("status") == "success"]
|
||||
if successful_results:
|
||||
print(f"\nGenerating HTML report...")
|
||||
report_file = generate_html_report(successful_results, args.output_dir)
|
||||
print(f" Report saved to: {report_file}")
|
||||
|
||||
# Also save summary JSON
|
||||
summary_file = args.output_dir / "comparison_summary.json"
|
||||
with open(summary_file, "w") as f:
|
||||
json.dump(
|
||||
{
|
||||
"timestamp": __import__("datetime").datetime.now().isoformat(),
|
||||
"presets_tested": [r["preset"] for r in results],
|
||||
"results": results,
|
||||
},
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
print(f" Summary saved to: {summary_file}")
|
||||
else:
|
||||
print(f"\nNote: No successful comparisons to report.")
|
||||
print(f" Capture files saved in {args.output_dir}")
|
||||
print(f" Run comparison when upstream files are available.")
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print("SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
status_counts = {}
|
||||
for result in results:
|
||||
status = result.get("status", "unknown")
|
||||
status_counts[status] = status_counts.get(status, 0) + 1
|
||||
|
||||
for status, count in sorted(status_counts.items()):
|
||||
print(f" {status}: {count}")
|
||||
|
||||
if "success" in status_counts:
|
||||
successful_results = [r for r in results if r.get("status") == "success"]
|
||||
avg_match = sum(
|
||||
r["stats"]["match_percentage"] for r in successful_results
|
||||
) / len(successful_results)
|
||||
print(f"\n Average match rate: {avg_match:.1f}%")
|
||||
|
||||
# Exit with error code if any failures
|
||||
if any(r.get("status") in ["error", "failed"] for r in results):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
341
tests/test_comparison_framework.py
Normal file
341
tests/test_comparison_framework.py
Normal file
@@ -0,0 +1,341 @@
|
||||
"""Comparison framework tests for upstream vs sideline pipeline.
|
||||
|
||||
These tests verify that the comparison framework works correctly
|
||||
and can be used for regression testing.
|
||||
"""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.comparison_capture import capture_frames, compare_captured_outputs
|
||||
|
||||
|
||||
class TestComparisonCapture:
|
||||
"""Tests for frame capture functionality."""
|
||||
|
||||
def test_capture_basic_preset(self):
|
||||
"""Test capturing frames from a basic preset."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
# Capture frames
|
||||
result = capture_frames(
|
||||
preset_name="comparison-basic",
|
||||
frame_count=10,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
# Verify result structure
|
||||
assert "preset" in result
|
||||
assert "config" in result
|
||||
assert "frames" in result
|
||||
assert "capture_stats" in result
|
||||
|
||||
# Verify frame count
|
||||
assert len(result["frames"]) == 10
|
||||
|
||||
# Verify frame structure
|
||||
frame = result["frames"][0]
|
||||
assert "frame_number" in frame
|
||||
assert "buffer" in frame
|
||||
assert "width" in frame
|
||||
assert "height" in frame
|
||||
|
||||
def test_capture_with_message_overlay(self):
|
||||
"""Test capturing frames with message overlay enabled."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
result = capture_frames(
|
||||
preset_name="comparison-with-message-overlay",
|
||||
frame_count=5,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
# Verify message overlay is enabled in config
|
||||
assert result["config"]["enable_message_overlay"] is True
|
||||
|
||||
def test_capture_multiple_presets(self):
|
||||
"""Test capturing frames from multiple presets."""
|
||||
presets = ["comparison-basic", "comparison-single-effect"]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
for preset in presets:
|
||||
result = capture_frames(
|
||||
preset_name=preset,
|
||||
frame_count=5,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
assert result["preset"] == preset
|
||||
|
||||
|
||||
class TestComparisonAnalysis:
|
||||
"""Tests for comparison analysis functionality."""
|
||||
|
||||
def test_compare_identical_outputs(self):
|
||||
"""Test comparing identical outputs shows 100% match."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
# Create two identical captured outputs
|
||||
sideline_file = output_dir / "test_sideline.json"
|
||||
upstream_file = output_dir / "test_upstream.json"
|
||||
|
||||
test_data = {
|
||||
"preset": "test",
|
||||
"config": {"viewport_width": 80, "viewport_height": 24},
|
||||
"frames": [
|
||||
{
|
||||
"frame_number": 0,
|
||||
"buffer": ["Line 1", "Line 2", "Line 3"],
|
||||
"width": 80,
|
||||
"height": 24,
|
||||
"render_time_ms": 10.0,
|
||||
}
|
||||
],
|
||||
"capture_stats": {
|
||||
"frame_count": 1,
|
||||
"total_time_ms": 10.0,
|
||||
"avg_frame_time_ms": 10.0,
|
||||
"fps": 100.0,
|
||||
},
|
||||
}
|
||||
|
||||
with open(sideline_file, "w") as f:
|
||||
json.dump(test_data, f)
|
||||
|
||||
with open(upstream_file, "w") as f:
|
||||
json.dump(test_data, f)
|
||||
|
||||
# Compare
|
||||
result = compare_captured_outputs(
|
||||
sideline_file=sideline_file,
|
||||
upstream_file=upstream_file,
|
||||
)
|
||||
|
||||
# Should have 100% match
|
||||
assert result["stats"]["match_percentage"] == 100.0
|
||||
assert result["stats"]["identical_frames"] == 1
|
||||
assert result["stats"]["total_differences"] == 0
|
||||
|
||||
def test_compare_different_outputs(self):
|
||||
"""Test comparing different outputs detects differences."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
sideline_file = output_dir / "test_sideline.json"
|
||||
upstream_file = output_dir / "test_upstream.json"
|
||||
|
||||
# Create different outputs
|
||||
sideline_data = {
|
||||
"preset": "test",
|
||||
"config": {"viewport_width": 80, "viewport_height": 24},
|
||||
"frames": [
|
||||
{
|
||||
"frame_number": 0,
|
||||
"buffer": ["Sideline Line 1", "Line 2"],
|
||||
"width": 80,
|
||||
"height": 24,
|
||||
"render_time_ms": 10.0,
|
||||
}
|
||||
],
|
||||
"capture_stats": {
|
||||
"frame_count": 1,
|
||||
"total_time_ms": 10.0,
|
||||
"avg_frame_time_ms": 10.0,
|
||||
"fps": 100.0,
|
||||
},
|
||||
}
|
||||
|
||||
upstream_data = {
|
||||
"preset": "test",
|
||||
"config": {"viewport_width": 80, "viewport_height": 24},
|
||||
"frames": [
|
||||
{
|
||||
"frame_number": 0,
|
||||
"buffer": ["Upstream Line 1", "Line 2"],
|
||||
"width": 80,
|
||||
"height": 24,
|
||||
"render_time_ms": 12.0,
|
||||
}
|
||||
],
|
||||
"capture_stats": {
|
||||
"frame_count": 1,
|
||||
"total_time_ms": 12.0,
|
||||
"avg_frame_time_ms": 12.0,
|
||||
"fps": 83.33,
|
||||
},
|
||||
}
|
||||
|
||||
with open(sideline_file, "w") as f:
|
||||
json.dump(sideline_data, f)
|
||||
|
||||
with open(upstream_file, "w") as f:
|
||||
json.dump(upstream_data, f)
|
||||
|
||||
# Compare
|
||||
result = compare_captured_outputs(
|
||||
sideline_file=sideline_file,
|
||||
upstream_file=upstream_file,
|
||||
)
|
||||
|
||||
# Should detect differences
|
||||
assert result["stats"]["match_percentage"] < 100.0
|
||||
assert result["stats"]["total_differences"] > 0
|
||||
assert len(result["frame_comparisons"][0]["line_diffs"]) > 0
|
||||
|
||||
def test_performance_comparison(self):
|
||||
"""Test that performance metrics are compared correctly."""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir)
|
||||
|
||||
sideline_file = output_dir / "test_sideline.json"
|
||||
upstream_file = output_dir / "test_upstream.json"
|
||||
|
||||
sideline_data = {
|
||||
"preset": "test",
|
||||
"config": {"viewport_width": 80, "viewport_height": 24},
|
||||
"frames": [
|
||||
{
|
||||
"frame_number": 0,
|
||||
"buffer": [],
|
||||
"width": 80,
|
||||
"height": 24,
|
||||
"render_time_ms": 10.0,
|
||||
}
|
||||
],
|
||||
"capture_stats": {
|
||||
"frame_count": 1,
|
||||
"total_time_ms": 10.0,
|
||||
"avg_frame_time_ms": 10.0,
|
||||
"fps": 100.0,
|
||||
},
|
||||
}
|
||||
|
||||
upstream_data = {
|
||||
"preset": "test",
|
||||
"config": {"viewport_width": 80, "viewport_height": 24},
|
||||
"frames": [
|
||||
{
|
||||
"frame_number": 0,
|
||||
"buffer": [],
|
||||
"width": 80,
|
||||
"height": 24,
|
||||
"render_time_ms": 12.0,
|
||||
}
|
||||
],
|
||||
"capture_stats": {
|
||||
"frame_count": 1,
|
||||
"total_time_ms": 12.0,
|
||||
"avg_frame_time_ms": 12.0,
|
||||
"fps": 83.33,
|
||||
},
|
||||
}
|
||||
|
||||
with open(sideline_file, "w") as f:
|
||||
json.dump(sideline_data, f)
|
||||
|
||||
with open(upstream_file, "w") as f:
|
||||
json.dump(upstream_data, f)
|
||||
|
||||
result = compare_captured_outputs(
|
||||
sideline_file=sideline_file,
|
||||
upstream_file=upstream_file,
|
||||
)
|
||||
|
||||
# Verify performance comparison
|
||||
perf = result["performance_comparison"]
|
||||
assert "sideline" in perf
|
||||
assert "upstream" in perf
|
||||
assert "diff" in perf
|
||||
assert (
|
||||
perf["sideline"]["fps"] > perf["upstream"]["fps"]
|
||||
) # Sideline is faster in this example
|
||||
|
||||
|
||||
class TestComparisonPresets:
|
||||
"""Tests for comparison preset configuration."""
|
||||
|
||||
def test_comparison_presets_exist(self):
|
||||
"""Test that comparison presets file exists and is valid."""
|
||||
presets_file = Path("tests/comparison_presets.toml")
|
||||
assert presets_file.exists(), "Comparison presets file should exist"
|
||||
|
||||
def test_preset_structure(self):
|
||||
"""Test that presets have required fields."""
|
||||
import tomli
|
||||
|
||||
with open("tests/comparison_presets.toml", "rb") as f:
|
||||
config = tomli.load(f)
|
||||
|
||||
presets = config.get("presets", {})
|
||||
assert len(presets) > 0, "Should have at least one preset"
|
||||
|
||||
for preset_name, preset_config in presets.items():
|
||||
# Each preset should have required fields
|
||||
assert "source" in preset_config, f"{preset_name} should have 'source'"
|
||||
assert "display" in preset_config, f"{preset_name} should have 'display'"
|
||||
assert "camera" in preset_config, f"{preset_name} should have 'camera'"
|
||||
assert "viewport_width" in preset_config, (
|
||||
f"{preset_name} should have 'viewport_width'"
|
||||
)
|
||||
assert "viewport_height" in preset_config, (
|
||||
f"{preset_name} should have 'viewport_height'"
|
||||
)
|
||||
assert "frame_count" in preset_config, (
|
||||
f"{preset_name} should have 'frame_count'"
|
||||
)
|
||||
|
||||
def test_preset_variety(self):
|
||||
"""Test that presets cover different scenarios."""
|
||||
import tomli
|
||||
|
||||
with open("tests/comparison_presets.toml", "rb") as f:
|
||||
config = tomli.load(f)
|
||||
|
||||
presets = config.get("presets", {})
|
||||
|
||||
# Should have presets for different categories
|
||||
categories = {
|
||||
"basic": 0,
|
||||
"effect": 0,
|
||||
"camera": 0,
|
||||
"source": 0,
|
||||
"viewport": 0,
|
||||
"comprehensive": 0,
|
||||
"regression": 0,
|
||||
}
|
||||
|
||||
for preset_name in presets.keys():
|
||||
name_lower = preset_name.lower()
|
||||
if "basic" in name_lower:
|
||||
categories["basic"] += 1
|
||||
elif (
|
||||
"effect" in name_lower or "border" in name_lower or "tint" in name_lower
|
||||
):
|
||||
categories["effect"] += 1
|
||||
elif "camera" in name_lower:
|
||||
categories["camera"] += 1
|
||||
elif "source" in name_lower:
|
||||
categories["source"] += 1
|
||||
elif (
|
||||
"viewport" in name_lower
|
||||
or "small" in name_lower
|
||||
or "large" in name_lower
|
||||
):
|
||||
categories["viewport"] += 1
|
||||
elif "comprehensive" in name_lower:
|
||||
categories["comprehensive"] += 1
|
||||
elif "regression" in name_lower:
|
||||
categories["regression"] += 1
|
||||
|
||||
# Verify we have variety
|
||||
assert categories["basic"] > 0, "Should have at least one basic preset"
|
||||
assert categories["effect"] > 0, "Should have at least one effect preset"
|
||||
assert categories["camera"] > 0, "Should have at least one camera preset"
|
||||
assert categories["source"] > 0, "Should have at least one source preset"
|
||||
Reference in New Issue
Block a user