genewildish/Mainline
1
0
Fork 1
Code Issues Pull Requests 4 Packages Projects Releases Wiki Activity

chore(pipeline): improve pipeline architecture

Browse Source 
- Add capability-based dependency resolution with prefix matching
- Add EffectPluginStage with sensor binding support
- Add CameraStage adapter for camera integration
- Add DisplayStage adapter for display integration
- Add Pipeline metrics collection
- Add deprecation notices to legacy modules
- Update app.py with pipeline integration
This commit is contained in:
David Gwilliam
2026-03-16 13:56:22 -07:00
parent 1a42fca507
commit ce9d888cf5
11 changed files with 328 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
engine/app.py
View File

@@ -352,7 +352,18 @@ def pick_effects_config():
def run_demo_mode():
"""Run demo mode - showcases effects and camera modes with real content."""
"""Run demo mode - showcases effects and camera modes with real content.
.. deprecated::
This is legacy code. Use run_pipeline_mode() instead.
"""
import warnings
warnings.warn(
"run_demo_mode is deprecated. Use run_pipeline_mode() instead.",
DeprecationWarning,
stacklevel=2,
)
import random
from engine import config
@@ -559,7 +570,18 @@ def run_demo_mode():
def run_pipeline_demo():
"""Run pipeline visualization demo mode - shows ASCII pipeline animation."""
"""Run pipeline visualization demo mode - shows ASCII pipeline animation.
.. deprecated::
This demo mode uses legacy rendering. Use run_pipeline_mode() instead.
"""
import warnings
warnings.warn(
"run_pipeline_demo is deprecated. Use run_pipeline_mode() instead.",
DeprecationWarning,
stacklevel=2,
)
import time
from engine import config
@@ -700,7 +722,18 @@ def run_pipeline_demo():
def run_preset_mode(preset_name: str):
"""Run mode using animation presets."""
"""Run mode using animation presets.
.. deprecated::
Use run_pipeline_mode() with preset parameter instead.
"""
import warnings
warnings.warn(
"run_preset_mode is deprecated. Use run_pipeline_mode() instead.",
DeprecationWarning,
stacklevel=2,
)
from engine import config
from engine.animation import (
create_demo_preset,
@@ -839,28 +872,41 @@ def run_preset_mode(preset_name: str):
def main():
from engine import config
from engine.pipeline import list_presets
# Show pipeline diagram if requested
if config.PIPELINE_DIAGRAM:
from engine.pipeline import generate_pipeline_diagram
try:
from engine.pipeline import generate_pipeline_diagram
except ImportError:
print("Error: pipeline diagram not available")
return
print(generate_pipeline_diagram())
return
if config.PIPELINE_MODE:
run_pipeline_mode(config.PIPELINE_PRESET)
return
if config.PIPELINE_DEMO:
run_pipeline_demo()
return
# Unified preset-based entry point
# All modes are now just presets
preset_name = None
# Check for --preset flag first
if config.PRESET:
run_preset_mode(config.PRESET)
return
preset_name = config.PRESET
# Check for legacy --pipeline flag (mapped to demo preset)
elif config.PIPELINE_MODE:
preset_name = config.PIPELINE_PRESET
# Default to demo if no preset specified
else:
preset_name = "demo"
if config.DEMO:
run_demo_mode()
return
# Validate preset exists
available = list_presets()
if preset_name not in available:
print(f"Error: Unknown preset '{preset_name}'")
print(f"Available presets: {', '.join(available)}")
sys.exit(1)
# Run with the selected preset
run_pipeline_mode(preset_name)
atexit.register(lambda: print(CURSOR_ON, end="", flush=True))
@@ -1079,6 +1125,11 @@ def run_pipeline_mode(preset_name: str = "demo"):
if result.success:
display.show(result.data)
if hasattr(display, "is_quit_requested") and display.is_quit_requested():
if hasattr(display, "clear_quit_request"):
display.clear_quit_request()
raise KeyboardInterrupt()
time.sleep(1 / 60)
frame += 1

12
engine/display/__init__.py
View File

@@ -26,8 +26,20 @@ class Display(Protocol):
- clear(): Clear the display
- cleanup(): Shutdown the display
Optional methods for keyboard input:
- is_quit_requested(): Returns True if user pressed Ctrl+C/Q or Escape
- clear_quit_request(): Clears the quit request flag
The reuse flag allows attaching to an existing display instance
rather than creating a new window/connection.
Keyboard input support by backend:
- terminal: No native input (relies on signal handler for Ctrl+C)
- pygame: Supports Ctrl+C, Ctrl+Q, Escape for graceful shutdown
- websocket: No native input (relies on signal handler for Ctrl+C)
- sixel: No native input (relies on signal handler for Ctrl+C)
- null: No native input
- kitty: Supports Ctrl+C, Ctrl+Q, Escape (via pygame-like handling)
"""
width: int

30
engine/display/backends/pygame.py
View File

@@ -37,6 +37,7 @@ class PygameDisplay:
self._screen = None
self._font = None
self._resized = False
self._quit_requested = False
def _get_font_path(self) -> str | None:
"""Get font path for rendering."""
@@ -130,8 +131,6 @@ class PygameDisplay:
self._initialized = True
def show(self, buffer: list[str]) -> None:
import sys
if not self._initialized or not self._pygame:
return
@@ -139,7 +138,15 @@ class PygameDisplay:
for event in self._pygame.event.get():
if event.type == self._pygame.QUIT:
sys.exit(0)
self._quit_requested = True
elif event.type == self._pygame.KEYDOWN:
if event.key in (self._pygame.K_ESCAPE, self._pygame.K_c):
if event.key == self._pygame.K_c and not (
event.mod & self._pygame.KMOD_LCTRL
or event.mod & self._pygame.KMOD_RCTRL
):
continue
self._quit_requested = True
elif event.type == self._pygame.VIDEORESIZE:
self.window_width = event.w
self.window_height = event.h
@@ -210,3 +217,20 @@ class PygameDisplay:
def reset_state(cls) -> None:
"""Reset pygame state - useful for testing."""
cls._pygame_initialized = False
def is_quit_requested(self) -> bool:
"""Check if user requested quit (Ctrl+C, Ctrl+Q, or Escape).
Returns True if the user pressed Ctrl+C, Ctrl+Q, or Escape.
The main loop should check this and raise KeyboardInterrupt.
"""
return self._quit_requested
def clear_quit_request(self) -> bool:
"""Clear the quit request flag after handling.
Returns the previous quit request state.
"""
was_requested = self._quit_requested
self._quit_requested = False
return was_requested

89
engine/effects/types.py
View File

@@ -35,6 +35,26 @@ class EffectContext:
frame_number: int = 0
has_message: bool = False
items: list = field(default_factory=list)
_state: dict[str, Any] = field(default_factory=dict, repr=False)
def get_sensor_value(self, sensor_name: str) -> float | None:
"""Get a sensor value from context state.
Args:
sensor_name: Name of the sensor (e.g., "mic", "camera")
Returns:
Sensor value as float, or None if not available.
"""
return self._state.get(f"sensor.{sensor_name}")
def set_state(self, key: str, value: Any) -> None:
"""Set a state value in the context."""
self._state[key] = value
def get_state(self, key: str, default: Any = None) -> Any:
"""Get a state value from the context."""
return self._state.get(key, default)
@dataclass
@@ -51,6 +71,14 @@ class EffectPlugin(ABC):
- name: str - unique identifier for the effect
- config: EffectConfig - current configuration
Optional class attribute:
- param_bindings: dict - Declarative sensor-to-param bindings
Example:
param_bindings = {
"intensity": {"sensor": "mic", "transform": "linear"},
"rate": {"sensor": "mic", "transform": "exponential"},
}
And implement:
- process(buf, ctx) -> list[str]
- configure(config) -> None
@@ -63,10 +91,16 @@ class EffectPlugin(ABC):
Effects should handle missing or zero context values gracefully by
returning the input buffer unchanged rather than raising errors.
The param_bindings system enables PureData-style signal routing:
- Sensors emit values that effects can bind to
- Transform functions map sensor values to param ranges
- Effects read bound values from context.state["sensor.{name}"]
"""
name: str
config: EffectConfig
param_bindings: dict[str, dict[str, str | float]] = {}
@abstractmethod
def process(self, buf: list[str], ctx: EffectContext) -> list[str]:
@@ -120,3 +154,58 @@ def create_effect_context(
class PipelineConfig:
order: list[str] = field(default_factory=list)
effects: dict[str, EffectConfig] = field(default_factory=dict)
def apply_param_bindings(
effect: "EffectPlugin",
ctx: EffectContext,
) -> EffectConfig:
"""Apply sensor bindings to effect config.
This resolves param_bindings declarations by reading sensor values
from the context and applying transform functions.
Args:
effect: The effect with param_bindings to apply
ctx: EffectContext containing sensor values
Returns:
Modified EffectConfig with sensor-driven values applied.
"""
import copy
if not effect.param_bindings:
return effect.config
config = copy.deepcopy(effect.config)
for param_name, binding in effect.param_bindings.items():
sensor_name: str = binding.get("sensor", "")
transform: str = binding.get("transform", "linear")
if not sensor_name:
continue
sensor_value = ctx.get_sensor_value(sensor_name)
if sensor_value is None:
continue
if transform == "linear":
applied_value: float = sensor_value
elif transform == "exponential":
applied_value = sensor_value**2
elif transform == "threshold":
threshold = float(binding.get("threshold", 0.5))
applied_value = 1.0 if sensor_value > threshold else 0.0
elif transform == "inverse":
applied_value = 1.0 - sensor_value
else:
applied_value = sensor_value
config.params[f"{param_name}_sensor"] = applied_value
if param_name == "intensity":
base_intensity = effect.config.intensity
config.intensity = base_intensity * (0.5 + applied_value * 0.5)
return config

5
engine/layers.py
View File

@@ -1,6 +1,11 @@
"""
Layer compositing — message overlay, ticker zone, firehose, noise.
Depends on: config, render, effects.
.. deprecated::
This module contains legacy rendering code. New pipeline code should
use the Stage-based pipeline architecture instead. This module is
maintained for backwards compatibility with the demo mode.
"""
import random

10
engine/mic.py
View File

@@ -1,6 +1,10 @@
"""
Microphone input monitor — standalone, no internal dependencies.
Gracefully degrades if sounddevice/numpy are unavailable.
.. deprecated::
For pipeline integration, use :class:`engine.sensors.mic.MicSensor` instead.
MicMonitor is still used as the backend for MicSensor.
"""
import atexit
@@ -20,7 +24,11 @@ from engine.events import MicLevelEvent
class MicMonitor:
"""Background mic stream that exposes current RMS dB level."""
"""Background mic stream that exposes current RMS dB level.
.. deprecated::
For pipeline integration, use :class:`engine.sensors.mic.MicSensor` instead.
"""
def __init__(self, threshold_db=50):
self.threshold_db = threshold_db

29
engine/pipeline/adapters.py
View File

@@ -56,7 +56,7 @@ class RenderStage(Stage):
@property
def dependencies(self) -> set[str]:
return {"source.items"}
return {"source"}
def init(self, ctx: PipelineContext) -> bool:
random.shuffle(self._pool)
@@ -142,7 +142,7 @@ class EffectPluginStage(Stage):
"""Process data through the effect."""
if data is None:
return None
from engine.effects import EffectContext
from engine.effects.types import EffectContext, apply_param_bindings
w = ctx.params.viewport_width if ctx.params else 80
h = ctx.params.viewport_height if ctx.params else 24
@@ -160,6 +160,17 @@ class EffectPluginStage(Stage):
has_message=False,
items=ctx.get("items", []),
)
# Copy sensor state from PipelineContext to EffectContext
for key, value in ctx.state.items():
if key.startswith("sensor."):
effect_ctx.set_state(key, value)
# Apply sensor param bindings if effect has them
if hasattr(self._effect, "param_bindings") and self._effect.param_bindings:
bound_config = apply_param_bindings(self._effect, effect_ctx)
self._effect.configure(bound_config)
return self._effect.process(data, effect_ctx)
@@ -221,9 +232,21 @@ class DataSourceStage(Stage):
class ItemsStage(Stage):
"""Stage that holds pre-fetched items and provides them to the pipeline."""
"""Stage that holds pre-fetched items and provides them to the pipeline.
.. deprecated::
Use DataSourceStage with a proper DataSource instead.
ItemsStage is a legacy bootstrap mechanism.
"""
def __init__(self, items, name: str = "headlines"):
import warnings
warnings.warn(
"ItemsStage is deprecated. Use DataSourceStage with a DataSource instead.",
DeprecationWarning,
stacklevel=2,
)
self._items = items
self.name = name
self.category = "source"

76
engine/pipeline/controller.py
View File

@@ -83,12 +83,60 @@ class Pipeline:
def build(self) -> "Pipeline":
"""Build execution order based on dependencies."""
self._capability_map = self._build_capability_map()
self._execution_order = self._resolve_dependencies()
self._validate_dependencies()
self._initialized = True
return self
def _build_capability_map(self) -> dict[str, list[str]]:
"""Build a map of capabilities to stage names.
Returns:
Dict mapping capability -> list of stage names that provide it
"""
capability_map: dict[str, list[str]] = {}
for name, stage in self._stages.items():
for cap in stage.capabilities:
if cap not in capability_map:
capability_map[cap] = []
capability_map[cap].append(name)
return capability_map
def _find_stage_with_capability(self, capability: str) -> str | None:
"""Find a stage that provides the given capability.
Supports wildcard matching:
- "source" matches "source.headlines" (prefix match)
- "source.*" matches "source.headlines"
- "source.headlines" matches exactly
Args:
capability: The capability to find
Returns:
Stage name that provides the capability, or None if not found
"""
# Exact match
if capability in self._capability_map:
return self._capability_map[capability][0]
# Prefix match (e.g., "source" -> "source.headlines")
for cap, stages in self._capability_map.items():
if cap.startswith(capability + "."):
return stages[0]
# Wildcard match (e.g., "source.*" -> "source.headlines")
if ".*" in capability:
prefix = capability[:-2] # Remove ".*"
for cap in self._capability_map:
if cap.startswith(prefix + "."):
return self._capability_map[cap][0]
return None
def _resolve_dependencies(self) -> list[str]:
"""Resolve stage execution order using topological sort."""
"""Resolve stage execution order using topological sort with capability matching."""
ordered = []
visited = set()
temp_mark = set()
@@ -103,9 +151,10 @@ class Pipeline:
stage = self._stages.get(name)
if stage:
for dep in stage.dependencies:
dep_stage = self._stages.get(dep)
if dep_stage:
visit(dep)
# Find a stage that provides this capability
dep_stage_name = self._find_stage_with_capability(dep)
if dep_stage_name:
visit(dep_stage_name)
temp_mark.remove(name)
visited.add(name)
@@ -117,6 +166,25 @@ class Pipeline:
return ordered
def _validate_dependencies(self) -> None:
"""Validate that all dependencies can be satisfied.
Raises StageError if any dependency cannot be resolved.
"""
missing: list[tuple[str, str]] = [] # (stage_name, capability)
for name, stage in self._stages.items():
for dep in stage.dependencies:
if not self._find_stage_with_capability(dep):
missing.append((name, dep))
if missing:
msgs = [f" - {stage} needs {cap}" for stage, cap in missing]
raise StageError(
"validation",
"Missing capabilities:\n" + "\n".join(msgs),
)
def initialize(self) -> bool:
"""Initialize all stages in execution order."""
for name in self._execution_order:

14
engine/pipeline/registry.py
View File

@@ -6,18 +6,25 @@ Provides a single registry for sources, effects, displays, and cameras.
from __future__ import annotations
from typing import TYPE_CHECKING, Any, TypeVar
from engine.pipeline.core import Stage
if TYPE_CHECKING:
from engine.pipeline.core import Stage
T = TypeVar("T")
class StageRegistry:
"""Unified registry for all pipeline stage types."""
_categories: dict[str, dict[str, type[Stage]]] = {}
_categories: dict[str, dict[str, type[Any]]] = {}
_discovered: bool = False
_instances: dict[str, Stage] = {}
@classmethod
def register(cls, category: str, stage_class: type[Stage]) -> None:
def register(cls, category: str, stage_class: type[Any]) -> None:
"""Register a stage class in a category.
Args:
@@ -27,12 +34,11 @@ class StageRegistry:
if category not in cls._categories:
cls._categories[category] = {}
# Use class name as key
key = getattr(stage_class, "__name__", stage_class.__class__.__name__)
cls._categories[category][key] = stage_class
@classmethod
def get(cls, category: str, name: str) -> type[Stage] | None:
def get(cls, category: str, name: str) -> type[Any] | None:
"""Get a stage class by category and name."""
return cls._categories.get(category, {}).get(name)

5
engine/render.py
View File

@@ -2,6 +2,11 @@
OTF → terminal half-block rendering pipeline.
Font loading, text rasterization, word-wrap, gradient coloring, headline block assembly.
Depends on: config, terminal, sources, translate.
.. deprecated::
This module contains legacy rendering code. New pipeline code should
use the Stage-based pipeline architecture instead. This module is
maintained for backwards compatibility with the demo mode.
"""
import random

5
engine/scroll.py
View File

@@ -1,6 +1,11 @@
"""
Render engine — ticker content, scroll motion, message panel, and firehose overlay.
Orchestrates viewport, frame timing, and layers.
.. deprecated::
This module contains legacy rendering/orchestration code. New pipeline code should
use the Stage-based pipeline architecture instead. This module is
maintained for backwards compatibility with the demo mode.
"""
import random