diff --git a/engine/app.py b/engine/app.py index 9048ad2..b5dec12 100644 --- a/engine/app.py +++ b/engine/app.py @@ -154,8 +154,12 @@ def run_pipeline_mode(preset_name: str = "demo"): # Add FontStage for headlines/poetry (default for demo) if preset.source in ["headlines", "poetry"]: - from engine.pipeline.adapters import FontStage + from engine.pipeline.adapters import FontStage, ViewportFilterStage + # Add viewport filter to prevent rendering all items + pipeline.add_stage( + "viewport_filter", ViewportFilterStage(name="viewport-filter") + ) pipeline.add_stage("font", FontStage(name="font")) else: # Fallback to simple conversion for other sources diff --git a/engine/display/__init__.py b/engine/display/__init__.py index 33d6394..e63fe82 100644 --- a/engine/display/__init__.py +++ b/engine/display/__init__.py @@ -84,6 +84,23 @@ class Display(Protocol): """ ... + def is_quit_requested(self) -> bool: + """Check if user requested quit (Ctrl+C, Ctrl+Q, or Escape). + + Returns: + True if quit was requested, False otherwise + + Optional method - only implemented by backends that support keyboard input. + """ + ... + + def clear_quit_request(self) -> None: + """Clear the quit request flag. + + Optional method - only implemented by backends that support keyboard input. + """ + ... + class DisplayRegistry: """Registry for display backends with auto-discovery.""" diff --git a/engine/pipeline/adapters.py b/engine/pipeline/adapters.py index ac9196c..dd2eafa 100644 --- a/engine/pipeline/adapters.py +++ b/engine/pipeline/adapters.py @@ -345,6 +345,64 @@ class CameraStage(Stage): self._camera.reset() +class ViewportFilterStage(Stage): + """Stage that limits items to fit in viewport. + + Filters the input list of items to only include as many as can fit + in the visible viewport. This prevents FontStage from rendering + thousands of items when only a few are visible, reducing processing time. + + Estimate: each rendered item typically takes 5-8 terminal lines. + For a 24-line viewport, we limit to ~4 items (conservative estimate). + """ + + def __init__(self, name: str = "viewport-filter"): + self.name = name + self.category = "filter" + self.optional = False + + @property + def stage_type(self) -> str: + return "filter" + + @property + def capabilities(self) -> set[str]: + return {f"filter.{self.name}"} + + @property + def dependencies(self) -> set[str]: + return {"source"} + + @property + def inlet_types(self) -> set: + from engine.pipeline.core import DataType + + return {DataType.SOURCE_ITEMS} + + @property + def outlet_types(self) -> set: + from engine.pipeline.core import DataType + + return {DataType.SOURCE_ITEMS} + + def process(self, data: Any, ctx: PipelineContext) -> Any: + """Filter items to viewport-fitting count.""" + if data is None or not isinstance(data, list): + return data + + # Estimate: each rendered headline takes 5-8 lines + # Use a conservative factor to ensure we don't run out of space + lines_per_item = 6 + viewport_height = ctx.params.viewport_height if ctx.params else 24 + + # Calculate how many items we need to fill the viewport + # Add 1 extra to account for padding/spacing + max_items = max(1, viewport_height // lines_per_item + 1) + + # Return only the items that fit in the viewport + return data[:max_items] + + class FontStage(Stage): """Stage that applies font rendering to content. diff --git a/engine/pipeline/controller.py b/engine/pipeline/controller.py index d72b7bd..3cf8e2c 100644 --- a/engine/pipeline/controller.py +++ b/engine/pipeline/controller.py @@ -255,6 +255,18 @@ class Pipeline: 1. Execute all non-overlay stages in dependency order 2. Apply overlay stages on top (sorted by render_order) """ + import os + import sys + + debug = os.environ.get("MAINLINE_DEBUG_DATAFLOW") == "1" + + if debug: + print( + f"[PIPELINE.execute] Starting with data type: {type(data).__name__ if data else 'None'}", + file=sys.stderr, + flush=True, + ) + if not self._initialized: self.build() @@ -303,8 +315,30 @@ class Pipeline: stage_start = time.perf_counter() if self._metrics_enabled else 0 try: + if debug: + data_info = type(current_data).__name__ + if isinstance(current_data, list): + data_info += f"[{len(current_data)}]" + print( + f"[STAGE.{name}] Starting with: {data_info}", + file=sys.stderr, + flush=True, + ) + current_data = stage.process(current_data, self.context) + + if debug: + data_info = type(current_data).__name__ + if isinstance(current_data, list): + data_info += f"[{len(current_data)}]" + print( + f"[STAGE.{name}] Completed, output: {data_info}", + file=sys.stderr, + flush=True, + ) except Exception as e: + if debug: + print(f"[STAGE.{name}] ERROR: {e}", file=sys.stderr, flush=True) if not stage.optional: return StageResult( success=False, diff --git a/pyproject.toml b/pyproject.toml index c7973ec..7929014 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ browser = [ ] dev = [ "pytest>=8.0.0", + "pytest-benchmark>=4.0.0", "pytest-cov>=4.1.0", "pytest-mock>=3.12.0", "ruff>=0.1.0", @@ -60,6 +61,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ "pytest>=8.0.0", + "pytest-benchmark>=4.0.0", "pytest-cov>=4.1.0", "pytest-mock>=3.12.0", "ruff>=0.1.0", diff --git a/tests/test_performance_regression.py b/tests/test_performance_regression.py new file mode 100644 index 0000000..6ffe845 --- /dev/null +++ b/tests/test_performance_regression.py @@ -0,0 +1,192 @@ +"""Performance regression tests for pipeline stages with realistic data volumes. + +These tests verify that the pipeline maintains performance with large datasets +by ensuring ViewportFilterStage prevents FontStage from rendering excessive items. + +Uses pytest-benchmark for statistical benchmarking with automatic regression detection. +""" + +import pytest + +from engine.data_sources.sources import SourceItem +from engine.pipeline.adapters import FontStage, ViewportFilterStage +from engine.pipeline.core import PipelineContext + + +class MockParams: + """Mock parameters object for testing.""" + + def __init__(self, viewport_width: int = 80, viewport_height: int = 24): + self.viewport_width = viewport_width + self.viewport_height = viewport_height + + +class TestViewportFilterPerformance: + """Test ViewportFilterStage performance with realistic data volumes.""" + + @pytest.mark.benchmark + def test_filter_2000_items_to_viewport(self, benchmark): + """Benchmark: Filter 2000 items to viewport size. + + Performance threshold: Must complete in < 1ms per iteration + This tests the filtering overhead is negligible. + """ + # Create 2000 test items (more than real headline sources) + test_items = [ + SourceItem(f"Headline {i}", f"source-{i % 10}", str(i)) for i in range(2000) + ] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=24) + + result = benchmark(stage.process, test_items, ctx) + + # Verify result is correct + assert len(result) <= 5 + assert len(result) > 0 + + @pytest.mark.benchmark + def test_font_stage_with_filtered_items(self, benchmark): + """Benchmark: FontStage rendering filtered (5) items. + + Performance threshold: Must complete in < 50ms per iteration + This tests that filtering saves significant time by reducing FontStage work. + """ + # Create filtered items (what ViewportFilterStage outputs) + filtered_items = [ + SourceItem(f"Headline {i}", "source", str(i)) + for i in range(5) # Filtered count + ] + + font_stage = FontStage() + ctx = PipelineContext() + ctx.params = MockParams() + + result = benchmark(font_stage.process, filtered_items, ctx) + + # Should render successfully + assert result is not None + assert isinstance(result, list) + assert len(result) > 0 + + def test_filter_reduces_work_by_288x(self): + """Verify ViewportFilterStage achieves expected performance improvement. + + With 1438 items and 24-line viewport: + - Without filter: FontStage renders all 1438 items + - With filter: FontStage renders ~5 items + - Expected improvement: 1438 / 5 ≈ 288x + """ + test_items = [ + SourceItem(f"Headline {i}", "source", str(i)) for i in range(1438) + ] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=24) + + filtered = stage.process(test_items, ctx) + improvement_factor = len(test_items) / len(filtered) + + # Verify we get expected 288x improvement + assert 250 < improvement_factor < 300 + # Verify filtered count is reasonable + assert 4 <= len(filtered) <= 6 + + +class TestPipelinePerformanceWithRealData: + """Integration tests for full pipeline performance with large datasets.""" + + def test_pipeline_handles_large_item_count(self): + """Test that pipeline doesn't hang with 2000+ items due to filtering.""" + # Create large dataset + large_items = [ + SourceItem(f"Headline {i}", f"source-{i % 5}", str(i)) for i in range(2000) + ] + + filter_stage = ViewportFilterStage() + font_stage = FontStage() + + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=24) + + # Filter should reduce items quickly + filtered = filter_stage.process(large_items, ctx) + assert len(filtered) < len(large_items) + + # FontStage should process filtered items quickly + rendered = font_stage.process(filtered, ctx) + assert rendered is not None + + def test_multiple_viewports_filter_correctly(self): + """Test that filter respects different viewport configurations.""" + large_items = [ + SourceItem(f"Headline {i}", "source", str(i)) for i in range(1000) + ] + + stage = ViewportFilterStage() + + # Test different viewport heights + test_cases = [ + (12, 3), # 12px height -> ~3 items + (24, 5), # 24px height -> ~5 items + (48, 9), # 48px height -> ~9 items + ] + + for viewport_height, expected_max_items in test_cases: + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=viewport_height) + + filtered = stage.process(large_items, ctx) + + # Verify filtering is proportional to viewport + assert len(filtered) <= expected_max_items + 1 + assert len(filtered) > 0 + + +class TestPerformanceRegressions: + """Tests that catch common performance regressions.""" + + def test_filter_doesnt_render_all_items(self): + """Regression test: Ensure filter doesn't accidentally render all items. + + This would indicate that ViewportFilterStage is broken or bypassed. + """ + large_items = [ + SourceItem(f"Headline {i}", "source", str(i)) for i in range(1438) + ] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams() + + filtered = stage.process(large_items, ctx) + + # Should NOT have all items (regression detection) + assert len(filtered) != len(large_items) + # Should have drastically fewer items + assert len(filtered) < 10 + + def test_font_stage_doesnt_hang_with_filter(self): + """Regression test: FontStage shouldn't hang when receiving filtered data. + + Previously, FontStage would render all items, causing 10+ second hangs. + Now it should receive only ~5 items and complete quickly. + """ + # Simulate what happens after ViewportFilterStage + filtered_items = [ + SourceItem(f"Headline {i}", "source", str(i)) + for i in range(5) # What filter outputs + ] + + font_stage = FontStage() + ctx = PipelineContext() + ctx.params = MockParams() + + # Should complete instantly (not hang) + result = font_stage.process(filtered_items, ctx) + + # Verify it actually worked + assert result is not None + assert isinstance(result, list) diff --git a/tests/test_viewport_filter_performance.py b/tests/test_viewport_filter_performance.py new file mode 100644 index 0000000..94f7ba7 --- /dev/null +++ b/tests/test_viewport_filter_performance.py @@ -0,0 +1,160 @@ +"""Integration tests for ViewportFilterStage with realistic data volumes. + +These tests verify that the ViewportFilterStage effectively reduces the number +of items processed by FontStage, preventing the 10+ second hangs observed with +large headline sources. +""" + + +from engine.data_sources.sources import SourceItem +from engine.pipeline.adapters import ViewportFilterStage +from engine.pipeline.core import PipelineContext + + +class MockParams: + """Mock parameters object for testing.""" + + def __init__(self, viewport_width: int = 80, viewport_height: int = 24): + self.viewport_width = viewport_width + self.viewport_height = viewport_height + + +class TestViewportFilterStage: + """Test ViewportFilterStage filtering behavior.""" + + def test_filter_stage_exists(self): + """Verify ViewportFilterStage can be instantiated.""" + stage = ViewportFilterStage() + assert stage is not None + assert stage.name == "viewport-filter" + + def test_filter_stage_properties(self): + """Verify ViewportFilterStage has correct type properties.""" + stage = ViewportFilterStage() + from engine.pipeline.core import DataType + + assert DataType.SOURCE_ITEMS in stage.inlet_types + assert DataType.SOURCE_ITEMS in stage.outlet_types + + def test_filter_large_item_count_to_viewport(self): + """Test filtering 1438 items (like real headlines) to viewport size.""" + # Create 1438 test items (matching real headline source) + test_items = [ + SourceItem(f"Headline {i}", f"source-{i % 5}", str(i)) for i in range(1438) + ] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams(viewport_width=80, viewport_height=24) + + # Filter items + filtered = stage.process(test_items, ctx) + + # Verify filtering reduced item count significantly + assert len(filtered) < len(test_items) + assert len(filtered) <= 5 # 24 height / 6 lines per item + 1 + assert len(filtered) > 0 # Must return at least 1 item + + def test_filter_respects_viewport_height(self): + """Test that filter respects different viewport heights.""" + test_items = [SourceItem(f"Headline {i}", "source", str(i)) for i in range(100)] + + stage = ViewportFilterStage() + + # Test with different viewport heights + for height in [12, 24, 48]: + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=height) + + filtered = stage.process(test_items, ctx) + expected_max = max(1, height // 6 + 1) + + assert len(filtered) <= expected_max + assert len(filtered) > 0 + + def test_filter_handles_empty_list(self): + """Test filter handles empty input gracefully.""" + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams() + + result = stage.process([], ctx) + + assert result == [] + + def test_filter_handles_none(self): + """Test filter handles None input gracefully.""" + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams() + + result = stage.process(None, ctx) + + assert result is None + + def test_filter_performance_improvement(self): + """Verify significant performance improvement (288x reduction).""" + # With 1438 items and 24-line viewport: + # - Without filter: FontStage renders all 1438 items + # - With filter: FontStage renders only ~5 items + # - Improvement: 1438 / 5 = 287.6x fewer items to render + + test_items = [ + SourceItem(f"Headline {i}", "source", str(i)) for i in range(1438) + ] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=24) + + filtered = stage.process(test_items, ctx) + improvement_factor = len(test_items) / len(filtered) + + # Verify we get at least 200x improvement + assert improvement_factor > 200 + # Verify we get the expected ~288x improvement + assert 250 < improvement_factor < 300 + + +class TestViewportFilterIntegration: + """Test ViewportFilterStage in pipeline context.""" + + def test_filter_output_is_source_items(self): + """Verify filter output can be consumed by FontStage.""" + from engine.pipeline.adapters import FontStage + + test_items = [ + SourceItem("Test Headline", "test-source", "123") for _ in range(10) + ] + + filter_stage = ViewportFilterStage() + font_stage = FontStage() + + ctx = PipelineContext() + ctx.params = MockParams() + + # Filter items + filtered = filter_stage.process(test_items, ctx) + + # Verify filtered output is compatible with FontStage + assert isinstance(filtered, list) + assert all(isinstance(item, SourceItem) for item in filtered) + + # FontStage should accept the filtered items + # (This would throw if types were incompatible) + result = font_stage.process(filtered, ctx) + assert result is not None + + def test_filter_preserves_item_order(self): + """Verify filter preserves order of first N items.""" + test_items = [SourceItem(f"Headline {i}", "source", str(i)) for i in range(20)] + + stage = ViewportFilterStage() + ctx = PipelineContext() + ctx.params = MockParams(viewport_height=24) + + filtered = stage.process(test_items, ctx) + + # Verify we kept the first N items in order + for i, item in enumerate(filtered): + assert item.content == f"Headline {i}"