klubhaus/sideline
2
0
Fork 0
Code Issues 5 Pull Requests Projects 1 Releases Wiki Activity

feature/capability-based-deps (#53)

Browse Source 
Reviewed-on: #53
Co-authored-by: David Gwilliam <dhgwilliam@gmail.com>
Co-committed-by: David Gwilliam <dhgwilliam@gmail.com>
This commit was merged in pull request #53.
This commit is contained in:
David Gwilliam
2026-03-31 01:55:21 +00:00
committed by david
parent 2650f7245e
commit 2d28e92594
190 changed files with 37860 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

341
tests/test_comparison_framework.py Normal file
View File

@@ -0,0 +1,341 @@
"""Comparison framework tests for upstream vs sideline pipeline.
These tests verify that the comparison framework works correctly
and can be used for regression testing.
"""
import json
import tempfile
from pathlib import Path
import pytest
from tests.comparison_capture import capture_frames, compare_captured_outputs
class TestComparisonCapture:
"""Tests for frame capture functionality."""
def test_capture_basic_preset(self):
"""Test capturing frames from a basic preset."""
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
# Capture frames
result = capture_frames(
preset_name="comparison-basic",
frame_count=10,
output_dir=output_dir,
)
# Verify result structure
assert "preset" in result
assert "config" in result
assert "frames" in result
assert "capture_stats" in result
# Verify frame count
assert len(result["frames"]) == 10
# Verify frame structure
frame = result["frames"][0]
assert "frame_number" in frame
assert "buffer" in frame
assert "width" in frame
assert "height" in frame
def test_capture_with_message_overlay(self):
"""Test capturing frames with message overlay enabled."""
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
result = capture_frames(
preset_name="comparison-with-message-overlay",
frame_count=5,
output_dir=output_dir,
)
# Verify message overlay is enabled in config
assert result["config"]["enable_message_overlay"] is True
def test_capture_multiple_presets(self):
"""Test capturing frames from multiple presets."""
presets = ["comparison-basic", "comparison-single-effect"]
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
for preset in presets:
result = capture_frames(
preset_name=preset,
frame_count=5,
output_dir=output_dir,
)
assert result["preset"] == preset
class TestComparisonAnalysis:
"""Tests for comparison analysis functionality."""
def test_compare_identical_outputs(self):
"""Test comparing identical outputs shows 100% match."""
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
# Create two identical captured outputs
sideline_file = output_dir / "test_sideline.json"
upstream_file = output_dir / "test_upstream.json"
test_data = {
"preset": "test",
"config": {"viewport_width": 80, "viewport_height": 24},
"frames": [
{
"frame_number": 0,
"buffer": ["Line 1", "Line 2", "Line 3"],
"width": 80,
"height": 24,
"render_time_ms": 10.0,
}
],
"capture_stats": {
"frame_count": 1,
"total_time_ms": 10.0,
"avg_frame_time_ms": 10.0,
"fps": 100.0,
},
}
with open(sideline_file, "w") as f:
json.dump(test_data, f)
with open(upstream_file, "w") as f:
json.dump(test_data, f)
# Compare
result = compare_captured_outputs(
sideline_file=sideline_file,
upstream_file=upstream_file,
)
# Should have 100% match
assert result["stats"]["match_percentage"] == 100.0
assert result["stats"]["identical_frames"] == 1
assert result["stats"]["total_differences"] == 0
def test_compare_different_outputs(self):
"""Test comparing different outputs detects differences."""
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
sideline_file = output_dir / "test_sideline.json"
upstream_file = output_dir / "test_upstream.json"
# Create different outputs
sideline_data = {
"preset": "test",
"config": {"viewport_width": 80, "viewport_height": 24},
"frames": [
{
"frame_number": 0,
"buffer": ["Sideline Line 1", "Line 2"],
"width": 80,
"height": 24,
"render_time_ms": 10.0,
}
],
"capture_stats": {
"frame_count": 1,
"total_time_ms": 10.0,
"avg_frame_time_ms": 10.0,
"fps": 100.0,
},
}
upstream_data = {
"preset": "test",
"config": {"viewport_width": 80, "viewport_height": 24},
"frames": [
{
"frame_number": 0,
"buffer": ["Upstream Line 1", "Line 2"],
"width": 80,
"height": 24,
"render_time_ms": 12.0,
}
],
"capture_stats": {
"frame_count": 1,
"total_time_ms": 12.0,
"avg_frame_time_ms": 12.0,
"fps": 83.33,
},
}
with open(sideline_file, "w") as f:
json.dump(sideline_data, f)
with open(upstream_file, "w") as f:
json.dump(upstream_data, f)
# Compare
result = compare_captured_outputs(
sideline_file=sideline_file,
upstream_file=upstream_file,
)
# Should detect differences
assert result["stats"]["match_percentage"] < 100.0
assert result["stats"]["total_differences"] > 0
assert len(result["frame_comparisons"][0]["line_diffs"]) > 0
def test_performance_comparison(self):
"""Test that performance metrics are compared correctly."""
with tempfile.TemporaryDirectory() as tmpdir:
output_dir = Path(tmpdir)
sideline_file = output_dir / "test_sideline.json"
upstream_file = output_dir / "test_upstream.json"
sideline_data = {
"preset": "test",
"config": {"viewport_width": 80, "viewport_height": 24},
"frames": [
{
"frame_number": 0,
"buffer": [],
"width": 80,
"height": 24,
"render_time_ms": 10.0,
}
],
"capture_stats": {
"frame_count": 1,
"total_time_ms": 10.0,
"avg_frame_time_ms": 10.0,
"fps": 100.0,
},
}
upstream_data = {
"preset": "test",
"config": {"viewport_width": 80, "viewport_height": 24},
"frames": [
{
"frame_number": 0,
"buffer": [],
"width": 80,
"height": 24,
"render_time_ms": 12.0,
}
],
"capture_stats": {
"frame_count": 1,
"total_time_ms": 12.0,
"avg_frame_time_ms": 12.0,
"fps": 83.33,
},
}
with open(sideline_file, "w") as f:
json.dump(sideline_data, f)
with open(upstream_file, "w") as f:
json.dump(upstream_data, f)
result = compare_captured_outputs(
sideline_file=sideline_file,
upstream_file=upstream_file,
)
# Verify performance comparison
perf = result["performance_comparison"]
assert "sideline" in perf
assert "upstream" in perf
assert "diff" in perf
assert (
perf["sideline"]["fps"] > perf["upstream"]["fps"]
) # Sideline is faster in this example
class TestComparisonPresets:
"""Tests for comparison preset configuration."""
def test_comparison_presets_exist(self):
"""Test that comparison presets file exists and is valid."""
presets_file = Path("tests/comparison_presets.toml")
assert presets_file.exists(), "Comparison presets file should exist"
def test_preset_structure(self):
"""Test that presets have required fields."""
import tomli
with open("tests/comparison_presets.toml", "rb") as f:
config = tomli.load(f)
presets = config.get("presets", {})
assert len(presets) > 0, "Should have at least one preset"
for preset_name, preset_config in presets.items():
# Each preset should have required fields
assert "source" in preset_config, f"{preset_name} should have 'source'"
assert "display" in preset_config, f"{preset_name} should have 'display'"
assert "camera" in preset_config, f"{preset_name} should have 'camera'"
assert "viewport_width" in preset_config, (
f"{preset_name} should have 'viewport_width'"
)
assert "viewport_height" in preset_config, (
f"{preset_name} should have 'viewport_height'"
)
assert "frame_count" in preset_config, (
f"{preset_name} should have 'frame_count'"
)
def test_preset_variety(self):
"""Test that presets cover different scenarios."""
import tomli
with open("tests/comparison_presets.toml", "rb") as f:
config = tomli.load(f)
presets = config.get("presets", {})
# Should have presets for different categories
categories = {
"basic": 0,
"effect": 0,
"camera": 0,
"source": 0,
"viewport": 0,
"comprehensive": 0,
"regression": 0,
}
for preset_name in presets.keys():
name_lower = preset_name.lower()
if "basic" in name_lower:
categories["basic"] += 1
elif (
"effect" in name_lower or "border" in name_lower or "tint" in name_lower
):
categories["effect"] += 1
elif "camera" in name_lower:
categories["camera"] += 1
elif "source" in name_lower:
categories["source"] += 1
elif (
"viewport" in name_lower
or "small" in name_lower
or "large" in name_lower
):
categories["viewport"] += 1
elif "comprehensive" in name_lower:
categories["comprehensive"] += 1
elif "regression" in name_lower:
categories["regression"] += 1
# Verify we have variety
assert categories["basic"] > 0, "Should have at least one basic preset"
assert categories["effect"] > 0, "Should have at least one effect preset"
assert categories["camera"] > 0, "Should have at least one camera preset"
assert categories["source"] > 0, "Should have at least one source preset"