forked from genewildish/Mainline
MAJOR REFACTORING: Consolidate duplicated pipeline code and standardize on capability-based dependency resolution. This is a significant but backwards-compatible restructuring that improves maintainability and extensibility. ## ARCHITECTURE CHANGES ### Data Sources Consolidation - Move engine/sources_v2.py → engine/data_sources/sources.py - Move engine/pipeline_sources/ → engine/data_sources/ - Create unified DataSource ABC with common interface: * fetch() - idempotent data retrieval * get_items() - cached access with automatic refresh * refresh() - force cache invalidation * is_dynamic - indicate streaming vs static sources - Support for SourceItem dataclass (content, source, timestamp, metadata) ### Display Backend Improvements - Update all 7 display backends to use new import paths - Terminal: Improve dimension detection and handling - WebSocket: Better error handling and client lifecycle - Sixel: Refactor graphics rendering - Pygame: Modernize event handling - Kitty: Add protocol support for inline images - Multi: Ensure proper forwarding to all backends - Null: Maintain testing backend functionality ### Pipeline Adapter Consolidation - Refactor adapter stages for clarity and flexibility - RenderStage now handles both item-based and buffer-based rendering - Add SourceItemsToBufferStage for converting data source items - Improve DataSourceStage to work with all source types - Add DisplayStage wrapper for display backends ### Camera & Viewport Refinements - Update Camera class for new architecture - Improve viewport dimension detection - Better handling of resize events across backends ### New Effect Plugins - border.py: Frame rendering effect with configurable style - crop.py: Viewport clipping effect for selective display - tint.py: Color filtering effect for atmosphere ### Tests & Quality - Add test_border_effect.py with comprehensive border tests - Add test_crop_effect.py with viewport clipping tests - Add test_tint_effect.py with color filtering tests - Update test_pipeline.py for new architecture - Update test_pipeline_introspection.py for new data source location - All 463 tests pass with 56% coverage - Linting: All checks pass with ruff ### Removals (Code Cleanup) - Delete engine/benchmark.py (deprecated performance testing) - Delete engine/pipeline_sources/__init__.py (moved to data_sources) - Delete engine/sources_v2.py (replaced by data_sources/sources.py) - Update AGENTS.md to reflect new structure ### Import Path Updates - Update engine/pipeline/controller.py::create_default_pipeline() * Old: from engine.sources_v2 import HeadlinesDataSource * New: from engine.data_sources.sources import HeadlinesDataSource - All display backends import from new locations - All tests import from new locations ## BACKWARDS COMPATIBILITY This refactoring is intended to be backwards compatible: - Pipeline execution unchanged (DAG-based with capability matching) - Effect plugins unchanged (EffectPlugin interface same) - Display protocol unchanged (Display duck-typing works as before) - Config system unchanged (presets.toml format same) ## TESTING - 463 tests pass (0 failures, 19 skipped) - Full linting check passes - Manual testing on demo, poetry, websocket modes - All new effect plugins tested ## FILES CHANGED - 24 files modified/added/deleted - 723 insertions, 1,461 deletions (net -738 LOC - cleanup!) - No breaking changes to public APIs - All transitive imports updated correctly
183 lines
4.7 KiB
Python
183 lines
4.7 KiB
Python
"""
|
|
Pipeline presets - Pre-configured pipeline configurations.
|
|
|
|
Provides PipelinePreset as a unified preset system.
|
|
Presets can be loaded from TOML files (presets.toml) or defined in code.
|
|
|
|
Loading order:
|
|
1. Built-in presets.toml in the package
|
|
2. User config ~/.config/mainline/presets.toml
|
|
3. Local ./presets.toml (overrides earlier)
|
|
"""
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
from engine.pipeline.params import PipelineParams
|
|
|
|
|
|
def _load_toml_presets() -> dict[str, Any]:
|
|
"""Load presets from TOML file."""
|
|
try:
|
|
from engine.pipeline.preset_loader import load_presets
|
|
|
|
return load_presets()
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
# Pre-load TOML presets
|
|
_YAML_PRESETS = _load_toml_presets()
|
|
|
|
|
|
@dataclass
|
|
class PipelinePreset:
|
|
"""Pre-configured pipeline with stages and animation.
|
|
|
|
A PipelinePreset packages:
|
|
- Initial params: Starting configuration
|
|
- Stages: List of stage configurations to create
|
|
|
|
This is the new unified preset that works with the Pipeline class.
|
|
"""
|
|
|
|
name: str
|
|
description: str = ""
|
|
source: str = "headlines"
|
|
display: str = "terminal"
|
|
camera: str = "vertical"
|
|
effects: list[str] = field(default_factory=list)
|
|
border: bool = False
|
|
|
|
def to_params(self) -> PipelineParams:
|
|
"""Convert to PipelineParams."""
|
|
params = PipelineParams()
|
|
params.source = self.source
|
|
params.display = self.display
|
|
params.border = self.border
|
|
params.camera_mode = self.camera
|
|
params.effect_order = self.effects.copy()
|
|
return params
|
|
|
|
@classmethod
|
|
def from_yaml(cls, name: str, data: dict[str, Any]) -> "PipelinePreset":
|
|
"""Create a PipelinePreset from YAML data."""
|
|
return cls(
|
|
name=name,
|
|
description=data.get("description", ""),
|
|
source=data.get("source", "headlines"),
|
|
display=data.get("display", "terminal"),
|
|
camera=data.get("camera", "vertical"),
|
|
effects=data.get("effects", []),
|
|
border=data.get("border", False),
|
|
)
|
|
|
|
|
|
# Built-in presets
|
|
DEMO_PRESET = PipelinePreset(
|
|
name="demo",
|
|
description="Demo mode with effect cycling and camera modes",
|
|
source="headlines",
|
|
display="pygame",
|
|
camera="vertical",
|
|
effects=["noise", "fade", "glitch", "firehose", "hud"],
|
|
)
|
|
|
|
POETRY_PRESET = PipelinePreset(
|
|
name="poetry",
|
|
description="Poetry feed with subtle effects",
|
|
source="poetry",
|
|
display="pygame",
|
|
camera="vertical",
|
|
effects=["fade", "hud"],
|
|
)
|
|
|
|
PIPELINE_VIZ_PRESET = PipelinePreset(
|
|
name="pipeline",
|
|
description="Pipeline visualization mode",
|
|
source="pipeline",
|
|
display="terminal",
|
|
camera="trace",
|
|
effects=["hud"],
|
|
)
|
|
|
|
WEBSOCKET_PRESET = PipelinePreset(
|
|
name="websocket",
|
|
description="WebSocket display mode",
|
|
source="headlines",
|
|
display="websocket",
|
|
camera="vertical",
|
|
effects=["noise", "fade", "glitch", "hud"],
|
|
)
|
|
|
|
SIXEL_PRESET = PipelinePreset(
|
|
name="sixel",
|
|
description="Sixel graphics display mode",
|
|
source="headlines",
|
|
display="sixel",
|
|
camera="vertical",
|
|
effects=["noise", "fade", "glitch", "hud"],
|
|
)
|
|
|
|
FIREHOSE_PRESET = PipelinePreset(
|
|
name="firehose",
|
|
description="High-speed firehose mode",
|
|
source="headlines",
|
|
display="pygame",
|
|
camera="vertical",
|
|
effects=["noise", "fade", "glitch", "firehose", "hud"],
|
|
)
|
|
|
|
|
|
# Build presets from YAML data
|
|
def _build_presets() -> dict[str, PipelinePreset]:
|
|
"""Build preset dictionary from all sources."""
|
|
result = {}
|
|
|
|
# Add YAML presets
|
|
yaml_presets = _YAML_PRESETS.get("presets", {})
|
|
for name, data in yaml_presets.items():
|
|
result[name] = PipelinePreset.from_yaml(name, data)
|
|
|
|
# Add built-in presets as fallback (if not in YAML)
|
|
builtins = {
|
|
"demo": DEMO_PRESET,
|
|
"poetry": POETRY_PRESET,
|
|
"pipeline": PIPELINE_VIZ_PRESET,
|
|
"websocket": WEBSOCKET_PRESET,
|
|
"sixel": SIXEL_PRESET,
|
|
"firehose": FIREHOSE_PRESET,
|
|
}
|
|
|
|
for name, preset in builtins.items():
|
|
if name not in result:
|
|
result[name] = preset
|
|
|
|
return result
|
|
|
|
|
|
PRESETS: dict[str, PipelinePreset] = _build_presets()
|
|
|
|
|
|
def get_preset(name: str) -> PipelinePreset | None:
|
|
"""Get a preset by name."""
|
|
return PRESETS.get(name)
|
|
|
|
|
|
def list_presets() -> list[str]:
|
|
"""List all available preset names."""
|
|
return list(PRESETS.keys())
|
|
|
|
|
|
def create_preset_from_params(
|
|
params: PipelineParams, name: str = "custom"
|
|
) -> PipelinePreset:
|
|
"""Create a preset from PipelineParams."""
|
|
return PipelinePreset(
|
|
name=name,
|
|
source=params.source,
|
|
display=params.display,
|
|
camera=params.camera_mode,
|
|
effects=params.effect_order.copy() if hasattr(params, "effect_order") else [],
|
|
)
|