forked from genewildish/Mainline
feat(pipeline): add unified pipeline architecture with Stage abstraction
- Add engine/pipeline/ module with Stage ABC, PipelineContext, PipelineParams - Stage provides unified interface for sources, effects, displays, cameras - Pipeline class handles DAG-based execution with dependency resolution - PipelinePreset for pre-configured pipelines (demo, poetry, pipeline, etc.) - Add PipelineParams as params layer for animation-driven config - Add StageRegistry for unified stage registration - Add sources_v2.py with DataSource.is_dynamic property - Add animation.py with Preset and AnimationController - Skip ntfy integration tests by default (require -m integration) - Skip e2e tests by default (require -m e2e) - Update pipeline.py with comprehensive introspection methods
This commit is contained in:
214
engine/sources_v2.py
Normal file
214
engine/sources_v2.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""
|
||||
Data source abstraction - Treat data sources as first-class citizens in the pipeline.
|
||||
|
||||
Each data source implements a common interface:
|
||||
- name: Display name for the source
|
||||
- fetch(): Fetch fresh data
|
||||
- stream(): Stream data continuously (optional)
|
||||
- get_items(): Get current items
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceItem:
|
||||
"""A single item from a data source."""
|
||||
|
||||
content: str
|
||||
source: str
|
||||
timestamp: str
|
||||
metadata: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class DataSource(ABC):
|
||||
"""Abstract base class for data sources.
|
||||
|
||||
Static sources: Data fetched once and cached. Safe to call fetch() multiple times.
|
||||
Dynamic sources: Data changes over time. fetch() should be idempotent.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Display name for this source."""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_dynamic(self) -> bool:
|
||||
"""Whether this source updates dynamically while the app runs. Default False."""
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
"""Fetch fresh data from the source. Must be idempotent."""
|
||||
...
|
||||
|
||||
def get_items(self) -> list[SourceItem]:
|
||||
"""Get current items. Default implementation returns cached fetch results."""
|
||||
if not hasattr(self, "_items") or self._items is None:
|
||||
self._items = self.fetch()
|
||||
return self._items
|
||||
|
||||
def refresh(self) -> list[SourceItem]:
|
||||
"""Force refresh - clear cache and fetch fresh data."""
|
||||
self._items = self.fetch()
|
||||
return self._items
|
||||
|
||||
def stream(self):
|
||||
"""Optional: Yield items continuously. Override for streaming sources."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __post_init__(self):
|
||||
self._items: list[SourceItem] | None = None
|
||||
|
||||
|
||||
class HeadlinesDataSource(DataSource):
|
||||
"""Data source for RSS feed headlines."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "headlines"
|
||||
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
from engine.fetch import fetch_all
|
||||
|
||||
items, _, _ = fetch_all()
|
||||
return [SourceItem(content=t, source=s, timestamp=ts) for t, s, ts in items]
|
||||
|
||||
|
||||
class PoetryDataSource(DataSource):
|
||||
"""Data source for Poetry DB."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "poetry"
|
||||
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
from engine.fetch import fetch_poetry
|
||||
|
||||
items, _, _ = fetch_poetry()
|
||||
return [SourceItem(content=t, source=s, timestamp=ts) for t, s, ts in items]
|
||||
|
||||
|
||||
class PipelineDataSource(DataSource):
|
||||
"""Data source for pipeline visualization (demo mode). Dynamic - updates every frame."""
|
||||
|
||||
def __init__(self, viewport_width: int = 80, viewport_height: int = 24):
|
||||
self.viewport_width = viewport_width
|
||||
self.viewport_height = viewport_height
|
||||
self.frame = 0
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "pipeline"
|
||||
|
||||
@property
|
||||
def is_dynamic(self) -> bool:
|
||||
return True
|
||||
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
from engine.pipeline_viz import generate_large_network_viewport
|
||||
|
||||
buffer = generate_large_network_viewport(
|
||||
self.viewport_width, self.viewport_height, self.frame
|
||||
)
|
||||
self.frame += 1
|
||||
content = "\n".join(buffer)
|
||||
return [
|
||||
SourceItem(content=content, source="pipeline", timestamp=f"f{self.frame}")
|
||||
]
|
||||
|
||||
def get_items(self) -> list[SourceItem]:
|
||||
return self.fetch()
|
||||
|
||||
|
||||
class CachedDataSource(DataSource):
|
||||
"""Data source that wraps another source with caching."""
|
||||
|
||||
def __init__(self, source: DataSource, max_items: int = 100):
|
||||
self.source = source
|
||||
self.max_items = max_items
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return f"cached:{self.source.name}"
|
||||
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
items = self.source.fetch()
|
||||
return items[: self.max_items]
|
||||
|
||||
def get_items(self) -> list[SourceItem]:
|
||||
if not hasattr(self, "_items") or self._items is None:
|
||||
self._items = self.fetch()
|
||||
return self._items
|
||||
|
||||
|
||||
class CompositeDataSource(DataSource):
|
||||
"""Data source that combines multiple sources."""
|
||||
|
||||
def __init__(self, sources: list[DataSource]):
|
||||
self.sources = sources
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "composite"
|
||||
|
||||
def fetch(self) -> list[SourceItem]:
|
||||
items = []
|
||||
for source in self.sources:
|
||||
items.extend(source.fetch())
|
||||
return items
|
||||
|
||||
|
||||
class SourceRegistry:
|
||||
"""Registry for data sources."""
|
||||
|
||||
def __init__(self):
|
||||
self._sources: dict[str, DataSource] = {}
|
||||
self._default: str | None = None
|
||||
|
||||
def register(self, source: DataSource, default: bool = False) -> None:
|
||||
self._sources[source.name] = source
|
||||
if default or self._default is None:
|
||||
self._default = source.name
|
||||
|
||||
def get(self, name: str) -> DataSource | None:
|
||||
return self._sources.get(name)
|
||||
|
||||
def list_all(self) -> dict[str, DataSource]:
|
||||
return dict(self._sources)
|
||||
|
||||
def default(self) -> DataSource | None:
|
||||
if self._default:
|
||||
return self._sources.get(self._default)
|
||||
return None
|
||||
|
||||
def create_headlines(self) -> HeadlinesDataSource:
|
||||
return HeadlinesDataSource()
|
||||
|
||||
def create_poetry(self) -> PoetryDataSource:
|
||||
return PoetryDataSource()
|
||||
|
||||
def create_pipeline(self, width: int = 80, height: int = 24) -> PipelineDataSource:
|
||||
return PipelineDataSource(width, height)
|
||||
|
||||
|
||||
_global_registry: SourceRegistry | None = None
|
||||
|
||||
|
||||
def get_source_registry() -> SourceRegistry:
|
||||
global _global_registry
|
||||
if _global_registry is None:
|
||||
_global_registry = SourceRegistry()
|
||||
return _global_registry
|
||||
|
||||
|
||||
def init_default_sources() -> SourceRegistry:
|
||||
"""Initialize the default source registry with standard sources."""
|
||||
registry = get_source_registry()
|
||||
registry.register(HeadlinesDataSource(), default=True)
|
||||
registry.register(PoetryDataSource())
|
||||
return registry
|
||||
Reference in New Issue
Block a user