forked from genewildish/Mainline
feat(pipeline): add unified pipeline architecture with Stage abstraction
- Add engine/pipeline/ module with Stage ABC, PipelineContext, PipelineParams - Stage provides unified interface for sources, effects, displays, cameras - Pipeline class handles DAG-based execution with dependency resolution - PipelinePreset for pre-configured pipelines (demo, poetry, pipeline, etc.) - Add PipelineParams as params layer for animation-driven config - Add StageRegistry for unified stage registration - Add sources_v2.py with DataSource.is_dynamic property - Add animation.py with Preset and AnimationController - Skip ntfy integration tests by default (require -m integration) - Skip e2e tests by default (require -m e2e) - Update pipeline.py with comprehensive introspection methods
This commit is contained in:
@@ -1,5 +1,23 @@
|
||||
"""
|
||||
Pipeline introspection - generates self-documenting diagrams of the render pipeline.
|
||||
|
||||
Pipeline Architecture:
|
||||
- Sources: Data providers (RSS, Poetry, Ntfy, Mic) - static or dynamic
|
||||
- Fetch: Retrieve data from sources
|
||||
- Prepare: Transform raw data (make_block, strip_tags, translate)
|
||||
- Scroll: Camera-based viewport rendering (ticker zone, message overlay)
|
||||
- Effects: Post-processing chain (noise, fade, glitch, firehose, hud)
|
||||
- Render: Final line rendering and layout
|
||||
- Display: Output backends (terminal, pygame, websocket, sixel, kitty)
|
||||
|
||||
Key abstractions:
|
||||
- DataSource: Sources can be static (cached) or dynamic (idempotent fetch)
|
||||
- Camera: Viewport controller (vertical, horizontal, omni, floating, trace)
|
||||
- EffectChain: Ordered effect processing pipeline
|
||||
- Display: Pluggable output backends
|
||||
- SourceRegistry: Source discovery and management
|
||||
- AnimationController: Time-based parameter animation
|
||||
- Preset: Package of initial params + animation for demo modes
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -33,8 +51,22 @@ class PipelineIntrospector:
|
||||
"""Generate a Mermaid flowchart of the pipeline."""
|
||||
lines = ["```mermaid", "flowchart TD"]
|
||||
|
||||
subgraph_groups = {
|
||||
"Sources": [],
|
||||
"Fetch": [],
|
||||
"Prepare": [],
|
||||
"Scroll": [],
|
||||
"Effects": [],
|
||||
"Display": [],
|
||||
"Async": [],
|
||||
"Animation": [],
|
||||
"Viz": [],
|
||||
}
|
||||
|
||||
other_nodes = []
|
||||
|
||||
for node in self.nodes:
|
||||
node_id = node.name.replace("-", "_").replace(" ", "_")
|
||||
node_id = node.name.replace("-", "_").replace(" ", "_").replace(":", "_")
|
||||
label = node.name
|
||||
if node.class_name:
|
||||
label = f"{node.name}\\n({node.class_name})"
|
||||
@@ -44,15 +76,55 @@ class PipelineIntrospector:
|
||||
if node.description:
|
||||
label += f"\\n{node.description}"
|
||||
|
||||
lines.append(f' {node_id}["{label}"]')
|
||||
node_entry = f' {node_id}["{label}"]'
|
||||
|
||||
if "DataSource" in node.name or "SourceRegistry" in node.name:
|
||||
subgraph_groups["Sources"].append(node_entry)
|
||||
elif "fetch" in node.name.lower():
|
||||
subgraph_groups["Fetch"].append(node_entry)
|
||||
elif (
|
||||
"make_block" in node.name
|
||||
or "strip_tags" in node.name
|
||||
or "translate" in node.name
|
||||
):
|
||||
subgraph_groups["Prepare"].append(node_entry)
|
||||
elif (
|
||||
"StreamController" in node.name
|
||||
or "render_ticker" in node.name
|
||||
or "render_message" in node.name
|
||||
or "Camera" in node.name
|
||||
):
|
||||
subgraph_groups["Scroll"].append(node_entry)
|
||||
elif "Effect" in node.name or "effect" in node.module:
|
||||
subgraph_groups["Effects"].append(node_entry)
|
||||
elif "Display:" in node.name:
|
||||
subgraph_groups["Display"].append(node_entry)
|
||||
elif "Ntfy" in node.name or "Mic" in node.name:
|
||||
subgraph_groups["Async"].append(node_entry)
|
||||
elif "Animation" in node.name or "Preset" in node.name:
|
||||
subgraph_groups["Animation"].append(node_entry)
|
||||
elif "pipeline_viz" in node.module or "CameraLarge" in node.name:
|
||||
subgraph_groups["Viz"].append(node_entry)
|
||||
else:
|
||||
other_nodes.append(node_entry)
|
||||
|
||||
for group_name, nodes in subgraph_groups.items():
|
||||
if nodes:
|
||||
lines.append(f" subgraph {group_name}")
|
||||
for node in nodes:
|
||||
lines.append(node)
|
||||
lines.append(" end")
|
||||
|
||||
for node in other_nodes:
|
||||
lines.append(node)
|
||||
|
||||
lines.append("")
|
||||
|
||||
for node in self.nodes:
|
||||
node_id = node.name.replace("-", "_").replace(" ", "_")
|
||||
node_id = node.name.replace("-", "_").replace(" ", "_").replace(":", "_")
|
||||
if node.inputs:
|
||||
for inp in node.inputs:
|
||||
inp_id = inp.replace("-", "_").replace(" ", "_")
|
||||
inp_id = inp.replace("-", "_").replace(" ", "_").replace(":", "_")
|
||||
lines.append(f" {inp_id} --> {node_id}")
|
||||
|
||||
lines.append("```")
|
||||
@@ -85,7 +157,8 @@ class PipelineIntrospector:
|
||||
lines.append(" Vertical --> Horizontal: set_mode()")
|
||||
lines.append(" Horizontal --> Omni: set_mode()")
|
||||
lines.append(" Omni --> Floating: set_mode()")
|
||||
lines.append(" Floating --> Vertical: set_mode()")
|
||||
lines.append(" Floating --> Trace: set_mode()")
|
||||
lines.append(" Trace --> Vertical: set_mode()")
|
||||
|
||||
lines.append(" state Vertical {")
|
||||
lines.append(" [*] --> ScrollUp")
|
||||
@@ -107,6 +180,11 @@ class PipelineIntrospector:
|
||||
lines.append(" Bobbing --> Bobbing: sin(time)")
|
||||
lines.append(" }")
|
||||
|
||||
lines.append(" state Trace {")
|
||||
lines.append(" [*] --> FollowPath")
|
||||
lines.append(" FollowPath --> FollowPath: node by node")
|
||||
lines.append(" }")
|
||||
|
||||
lines.append("```")
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -144,6 +222,71 @@ class PipelineIntrospector:
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_sources_v2(self) -> None:
|
||||
"""Introspect data sources v2 (new abstraction)."""
|
||||
from engine.sources_v2 import SourceRegistry, init_default_sources
|
||||
|
||||
init_default_sources()
|
||||
SourceRegistry()
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="SourceRegistry",
|
||||
module="engine.sources_v2",
|
||||
class_name="SourceRegistry",
|
||||
description="Source discovery and management",
|
||||
)
|
||||
)
|
||||
|
||||
for name, desc in [
|
||||
("HeadlinesDataSource", "RSS feed headlines"),
|
||||
("PoetryDataSource", "Poetry DB"),
|
||||
("PipelineDataSource", "Pipeline viz (dynamic)"),
|
||||
]:
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name=f"DataSource: {name}",
|
||||
module="engine.sources_v2",
|
||||
class_name=name,
|
||||
description=f"{desc}",
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_prepare(self) -> None:
|
||||
"""Introspect prepare layer (transformation)."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="make_block",
|
||||
module="engine.render",
|
||||
func_name="make_block",
|
||||
description="Transform headline into display block",
|
||||
inputs=["title", "source", "timestamp", "width"],
|
||||
outputs=["block"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="strip_tags",
|
||||
module="engine.filter",
|
||||
func_name="strip_tags",
|
||||
description="Remove HTML tags from content",
|
||||
inputs=["html"],
|
||||
outputs=["plain_text"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="translate_headline",
|
||||
module="engine.translate",
|
||||
func_name="translate_headline",
|
||||
description="Translate headline to target language",
|
||||
inputs=["title", "target_lang"],
|
||||
outputs=["translated_title"],
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_fetch(self) -> None:
|
||||
"""Introspect fetch layer."""
|
||||
self.add_node(
|
||||
@@ -190,6 +333,121 @@ class PipelineIntrospector:
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="render_message_overlay",
|
||||
module="engine.layers",
|
||||
func_name="render_message_overlay",
|
||||
description="Render ntfy message overlay",
|
||||
inputs=["msg", "width", "height"],
|
||||
outputs=["overlay", "cache"],
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_render(self) -> None:
|
||||
"""Introspect render layer."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="big_wrap",
|
||||
module="engine.render",
|
||||
func_name="big_wrap",
|
||||
description="Word-wrap text to width",
|
||||
inputs=["text", "width"],
|
||||
outputs=["lines"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="lr_gradient",
|
||||
module="engine.render",
|
||||
func_name="lr_gradient",
|
||||
description="Apply left-right gradient to lines",
|
||||
inputs=["lines", "position"],
|
||||
outputs=["styled_lines"],
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_async_sources(self) -> None:
|
||||
"""Introspect async data sources (ntfy, mic)."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="NtfyPoller",
|
||||
module="engine.ntfy",
|
||||
class_name="NtfyPoller",
|
||||
description="Poll ntfy for messages (async)",
|
||||
inputs=["topic"],
|
||||
outputs=["message"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="MicMonitor",
|
||||
module="engine.mic",
|
||||
class_name="MicMonitor",
|
||||
description="Monitor microphone input (async)",
|
||||
outputs=["audio_level"],
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_eventbus(self) -> None:
|
||||
"""Introspect event bus for decoupled communication."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="EventBus",
|
||||
module="engine.eventbus",
|
||||
class_name="EventBus",
|
||||
description="Thread-safe event publishing",
|
||||
inputs=["event"],
|
||||
outputs=["subscribers"],
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_animation(self) -> None:
|
||||
"""Introspect animation system."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="AnimationController",
|
||||
module="engine.animation",
|
||||
class_name="AnimationController",
|
||||
description="Time-based parameter animation",
|
||||
inputs=["dt"],
|
||||
outputs=["params"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="Preset",
|
||||
module="engine.animation",
|
||||
class_name="Preset",
|
||||
description="Package of initial params + animation",
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_pipeline_viz(self) -> None:
|
||||
"""Introspect pipeline visualization."""
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="generate_large_network_viewport",
|
||||
module="engine.pipeline_viz",
|
||||
func_name="generate_large_network_viewport",
|
||||
description="Large animated network visualization",
|
||||
inputs=["viewport_w", "viewport_h", "frame"],
|
||||
outputs=["buffer"],
|
||||
)
|
||||
)
|
||||
|
||||
self.add_node(
|
||||
PipelineNode(
|
||||
name="CameraLarge",
|
||||
module="engine.pipeline_viz",
|
||||
class_name="CameraLarge",
|
||||
description="Large grid camera (trace mode)",
|
||||
)
|
||||
)
|
||||
|
||||
def introspect_camera(self) -> None:
|
||||
"""Introspect camera system."""
|
||||
self.add_node(
|
||||
@@ -246,11 +504,18 @@ class PipelineIntrospector:
|
||||
def run(self) -> str:
|
||||
"""Run full introspection."""
|
||||
self.introspect_sources()
|
||||
self.introspect_sources_v2()
|
||||
self.introspect_fetch()
|
||||
self.introspect_prepare()
|
||||
self.introspect_scroll()
|
||||
self.introspect_render()
|
||||
self.introspect_camera()
|
||||
self.introspect_effects()
|
||||
self.introspect_display()
|
||||
self.introspect_async_sources()
|
||||
self.introspect_eventbus()
|
||||
self.introspect_animation()
|
||||
self.introspect_pipeline_viz()
|
||||
|
||||
return self.generate_full_diagram()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user