From dcc37180128c70cceb9a0d0594acda2504d575cf Mon Sep 17 00:00:00 2001 From: Gene Johnson Date: Sat, 14 Mar 2026 20:56:24 -0700 Subject: [PATCH] refactor: Create `engine` package, extracting data sources to `sources.py`, and add refactoring documentation. --- Refactor mainline.md | 178 +++++++++++++++++++++++++++++++++++++++++++ engine/__init__.py | 1 + engine/config.py | 31 ++++++++ engine/sources.py | 115 ++++++++++++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 Refactor mainline.md create mode 100644 engine/__init__.py create mode 100644 engine/config.py create mode 100644 engine/sources.py diff --git a/Refactor mainline.md b/Refactor mainline.md new file mode 100644 index 0000000..467c590 --- /dev/null +++ b/Refactor mainline.md @@ -0,0 +1,178 @@ +# Refactor mainline\.py into modular package +## Problem +`mainline.py` is a single 1085\-line file with ~10 interleaved concerns\. This prevents: +* Reusing the ntfy doorbell interrupt in other visualizers +* Importing the render pipeline from `serve.py` \(future ESP32 HTTP server\) +* Testing any concern in isolation +* Porting individual layers to Rust independently +## Target structure +```warp-runnable-command +mainline.py # thin entrypoint: venv bootstrap → engine.app.main() +engine/ + __init__.py + config.py # constants, CLI flags, glyph tables + sources.py # FEEDS, POETRY_SOURCES, SOURCE_LANGS, _LOCATION_LANGS + terminal.py # ANSI codes, tw/th, type_out, slow_print, boot_ln + filter.py # HTML stripping, content filter (_SKIP_RE) + translate.py # Google Translate wrapper + location→language detection + render.py # OTF font loading, _render_line, _big_wrap, _lr_gradient, _make_block + effects.py # noise, glitch_bar, _fade_line, _vis_trunc, _firehose_line, _next_headline + fetch.py # RSS/Gutenberg fetching, cache load/save + ntfy.py # NtfyPoller class — standalone, zero internal deps + mic.py # MicMonitor class — standalone + scroll.py # stream() frame loop + message rendering + app.py # main(), TITLE art, boot sequence, signal handler +``` +The package is named `engine/` to avoid a naming conflict with the `mainline.py` entrypoint\. +## Module dependency graph +```warp-runnable-command +config ← (nothing) +sources ← (nothing) +terminal ← (nothing) +filter ← (nothing) +translate ← sources +render ← config, terminal, sources +effects ← config, terminal, sources +fetch ← config, sources, filter, terminal +ntfy ← (nothing — stdlib only, fully standalone) +mic ← (nothing — sounddevice only) +scroll ← config, terminal, render, effects, ntfy, mic +app ← everything above +``` +Critical property: **ntfy\.py and mic\.py have zero internal dependencies**, making ntfy reusable by any visualizer\. +## Module details +### mainline\.py \(entrypoint — slimmed down\) +Keeps only the venv bootstrap \(lines 10\-38\) which must run before any third\-party imports\. After bootstrap, delegates to `engine.app.main()`\. +### engine/config\.py +From current mainline\.py: +* `HEADLINE_LIMIT`, `FEED_TIMEOUT`, `MIC_THRESHOLD_DB` \(lines 55\-57\) +* `MODE`, `FIREHOSE` CLI flag parsing \(lines 58\-59\) +* `NTFY_TOPIC`, `NTFY_POLL_INTERVAL`, `MESSAGE_DISPLAY_SECS` \(lines 62\-64\) +* `_FONT_PATH`, `_FONT_SZ`, `_RENDER_H` \(lines 147\-150\) +* `_SCROLL_DUR`, `_FRAME_DT`, `FIREHOSE_H` \(lines 505\-507\) +* `GLITCH`, `KATA` glyph tables \(lines 143\-144\) +### engine/sources\.py +Pure data, no logic: +* `FEEDS` dict \(lines 102\-140\) +* `POETRY_SOURCES` dict \(lines 67\-80\) +* `SOURCE_LANGS` dict \(lines 258\-266\) +* `_LOCATION_LANGS` dict \(lines 269\-289\) +* `_SCRIPT_FONTS` dict \(lines 153\-165\) +* `_NO_UPPER` set \(line 167\) +### engine/terminal\.py +ANSI primitives and terminal I/O: +* All ANSI constants: `RST`, `BOLD`, `DIM`, `G_HI`, `G_MID`, `G_LO`, `G_DIM`, `W_COOL`, `W_DIM`, `W_GHOST`, `C_DIM`, `CLR`, `CURSOR_OFF`, `CURSOR_ON` \(lines 83\-99\) +* `tw()`, `th()` \(lines 223\-234\) +* `type_out()`, `slow_print()`, `boot_ln()` \(lines 355\-386\) +### engine/filter\.py +* `_Strip` HTML parser class \(lines 205\-214\) +* `strip_tags()` \(lines 217\-220\) +* `_SKIP_RE` compiled regex \(lines 322\-346\) +* `_skip()` predicate \(lines 349\-351\) +### engine/translate\.py +* `_TRANSLATE_CACHE` \(line 291\) +* `_detect_location_language()` \(lines 294\-300\) — imports `_LOCATION_LANGS` from sources +* `_translate_headline()` \(lines 303\-319\) +### engine/render\.py +The OTF→terminal pipeline\. This is exactly what `serve.py` will import to produce 1\-bit bitmaps for the ESP32\. +* `_GRAD_COLS` gradient table \(lines 169\-182\) +* `_font()`, `_font_for_lang()` with lazy\-load \+ cache \(lines 185\-202\) +* `_render_line()` — OTF text → half\-block terminal rows \(lines 567\-605\) +* `_big_wrap()` — word\-wrap \+ render \(lines 608\-636\) +* `_lr_gradient()` — apply left→right color gradient \(lines 639\-656\) +* `_make_block()` — composite: translate → render → colorize a headline \(lines 718\-756\)\. Imports from translate, sources\. +### engine/effects\.py +Visual effects applied during the frame loop: +* `noise()` \(lines 237\-245\) +* `glitch_bar()` \(lines 248\-252\) +* `_fade_line()` — probabilistic character dissolve \(lines 659\-680\) +* `_vis_trunc()` — ANSI\-aware width truncation \(lines 683\-701\) +* `_firehose_line()` \(lines 759\-801\) — imports config\.MODE, sources\.FEEDS/POETRY\_SOURCES +* `_next_headline()` — pool management \(lines 704\-715\) +### engine/fetch\.py +* `fetch_feed()` \(lines 390\-396\) +* `fetch_all()` \(lines 399\-426\) — imports filter\.\_skip, filter\.strip\_tags, terminal\.boot\_ln +* `_fetch_gutenberg()` \(lines 429\-456\) +* `fetch_poetry()` \(lines 459\-472\) +* `_cache_path()`, `_load_cache()`, `_save_cache()` \(lines 476\-501\) +### engine/ntfy\.py — standalone, reusable +Refactored from the current globals \+ thread \(lines 531\-564\) and the message rendering section of `stream()` \(lines 845\-909\) into a class: +```python +class NtfyPoller: + def __init__(self, topic_url, poll_interval=15, display_secs=30): + ... + def start(self): + """Start background polling thread.""" + def get_active_message(self): + """Return (title, body, timestamp) if a message is active and not expired, else None.""" + def dismiss(self): + """Manually dismiss current message.""" +``` +Dependencies: `urllib.request`, `json`, `threading`, `time` — all stdlib\. No internal imports\. +Other visualizers use it like: +```python +from engine.ntfy import NtfyPoller +poller = NtfyPoller("https://ntfy.sh/my_topic/json?since=20s&poll=1") +poller.start() +# in render loop: +msg = poller.get_active_message() +if msg: + title, body, ts = msg + render_my_message(title, body) # visualizer-specific +``` +### engine/mic\.py — standalone +Refactored from the current globals \(lines 508\-528\) into a class: +```python +class MicMonitor: + def __init__(self, threshold_db=50): + ... + def start(self) -> bool: + """Start background mic stream. Returns False if unavailable.""" + def stop(self): + ... + @property + def db(self) -> float: + """Current RMS dB level.""" + @property + def excess(self) -> float: + """dB above threshold (clamped to 0).""" +``` +Dependencies: `sounddevice`, `numpy` \(both optional — graceful fallback\)\. +### engine/scroll\.py +The `stream()` function \(lines 804\-990\)\. Receives its dependencies via arguments or imports: +* `stream(items, ntfy_poller, mic_monitor, config)` or similar +* Message rendering \(lines 855\-909\) stays here since it's terminal\-display\-specific — a different visualizer would render messages differently +### engine/app\.py +The orchestrator: +* `TITLE` ASCII art \(lines 994\-1001\) +* `main()` \(lines 1004\-1084\): CLI handling, signal setup, boot animation, fetch, wire up ntfy/mic/scroll +## Execution order +### Step 1: Create engine/ package skeleton +Create `engine/__init__.py` and all empty module files\. +### Step 2: Extract pure data modules \(zero\-dep\) +Move constants and data dicts into `config.py`, `sources.py`\. These have no logic dependencies\. +### Step 3: Extract terminal\.py +Move ANSI codes and terminal I/O helpers\. No internal deps\. +### Step 4: Extract filter\.py and translate\.py +Both are small, self\-contained\. translate imports from sources\. +### Step 5: Extract render\.py +Font loading \+ the OTF→half\-block pipeline\. Imports from config, terminal, sources\. This is the module `serve.py` will later import\. +### Step 6: Extract effects\.py +Visual effects\. Imports from config, terminal, sources\. +### Step 7: Extract fetch\.py +Feed/Gutenberg fetching \+ caching\. Imports from config, sources, filter, terminal\. +### Step 8: Extract ntfy\.py and mic\.py +Refactor globals\+threads into classes\. Zero internal deps\. +### Step 9: Extract scroll\.py +The frame loop\. Last to extract because it depends on everything above\. +### Step 10: Extract app\.py +The `main()` function, boot sequence, signal handler\. Wire up all modules\. +### Step 11: Slim down mainline\.py +Keep only venv bootstrap \+ `from engine.app import main; main()`\. +### Step 12: Verify +Run `python3 mainline.py`, `python3 mainline.py --poetry`, and `python3 mainline.py --firehose` to confirm identical behavior\. No behavioral changes in this refactor\. +## What this enables +* **serve\.py** \(future\): `from engine.render import _render_line, _big_wrap` \+ `from engine.fetch import fetch_all` — imports the pipeline directly +* **Other visualizers**: `from engine.ntfy import NtfyPoller` — doorbell feature with no coupling to mainline's scroll engine +* **Rust port**: Clear boundaries for what to port first \(ntfy client, render pipeline\) vs what stays in Python \(fetching, caching — the server side\) +* **Testing**: Each module can be unit\-tested in isolation diff --git a/engine/__init__.py b/engine/__init__.py new file mode 100644 index 0000000..63f007f --- /dev/null +++ b/engine/__init__.py @@ -0,0 +1 @@ +# engine — modular internals for mainline diff --git a/engine/config.py b/engine/config.py new file mode 100644 index 0000000..dcf762e --- /dev/null +++ b/engine/config.py @@ -0,0 +1,31 @@ +""" +Configuration constants, CLI flags, and glyph tables. +""" + +import sys + +# ─── RUNTIME ────────────────────────────────────────────── +HEADLINE_LIMIT = 1000 +FEED_TIMEOUT = 10 +MIC_THRESHOLD_DB = 50 # dB above which glitches intensify +MODE = 'poetry' if '--poetry' in sys.argv or '-p' in sys.argv else 'news' +FIREHOSE = '--firehose' in sys.argv + +# ─── NTFY MESSAGE QUEUE ────────────────────────────────── +NTFY_TOPIC = "https://ntfy.sh/klubhaus_terminal_mainline/json?since=20s&poll=1" +NTFY_POLL_INTERVAL = 15 # seconds between polls +MESSAGE_DISPLAY_SECS = 30 # how long a message holds the screen + +# ─── FONT RENDERING ────────────────────────────────────── +FONT_PATH = "/Users/genejohnson/Documents/CS Bishop Drawn/CSBishopDrawn-Italic.otf" +FONT_SZ = 60 +RENDER_H = 8 # terminal rows per rendered text line + +# ─── SCROLL / FRAME ────────────────────────────────────── +SCROLL_DUR = 3.75 # seconds per headline +FRAME_DT = 0.05 # 50ms base frame rate (20 FPS) +FIREHOSE_H = 12 # firehose zone height (terminal rows) + +# ─── GLYPHS ─────────────────────────────────────────────── +GLITCH = "░▒▓█▌▐╌╍╎╏┃┆┇┊┋" +KATA = "ﾊﾐﾋｰｳｼﾅﾓﾆｻﾜﾂｵﾘｱﾎﾃﾏｹﾒｴｶｷﾑﾕﾗｾﾈｽﾀﾇﾍ" diff --git a/engine/sources.py b/engine/sources.py new file mode 100644 index 0000000..d7b6733 --- /dev/null +++ b/engine/sources.py @@ -0,0 +1,115 @@ +""" +Data sources: feed URLs, poetry sources, language mappings, script fonts. +Pure data — no logic, no dependencies. +""" + +# ─── RSS FEEDS ──────────────────────────────────────────── +FEEDS = { + # Science & Technology + "Nature": "https://www.nature.com/nature.rss", + "Science Daily": "https://www.sciencedaily.com/rss/all.xml", + "Phys.org": "https://phys.org/rss-feed/", + "NASA": "https://www.nasa.gov/news-release/feed/", + "Ars Technica": "https://feeds.arstechnica.com/arstechnica/index", + "New Scientist": "https://www.newscientist.com/section/news/feed/", + "Quanta": "https://api.quantamagazine.org/feed/", + "BBC Science": "http://feeds.bbci.co.uk/news/science_and_environment/rss.xml", + "MIT Tech Review": "https://www.technologyreview.com/feed/", + # Economics & Business + "BBC Business": "http://feeds.bbci.co.uk/news/business/rss.xml", + "MarketWatch": "https://feeds.marketwatch.com/marketwatch/topstories/", + "Economist": "https://www.economist.com/finance-and-economics/rss.xml", + # World & Politics + "BBC World": "http://feeds.bbci.co.uk/news/world/rss.xml", + "NPR": "https://feeds.npr.org/1001/rss.xml", + "Al Jazeera": "https://www.aljazeera.com/xml/rss/all.xml", + "Guardian World": "https://www.theguardian.com/world/rss", + "DW": "https://rss.dw.com/rdf/rss-en-all", + "France24": "https://www.france24.com/en/rss", + "ABC Australia": "https://www.abc.net.au/news/feed/2942460/rss.xml", + "Japan Times": "https://www.japantimes.co.jp/feed/", + "The Hindu": "https://www.thehindu.com/news/national/feeder/default.rss", + "SCMP": "https://www.scmp.com/rss/91/feed", + "Der Spiegel": "https://www.spiegel.de/international/index.rss", + # Culture & Ideas + "Guardian Culture": "https://www.theguardian.com/culture/rss", + "Aeon": "https://aeon.co/feed.rss", + "Smithsonian": "https://www.smithsonianmag.com/rss/latest_articles/", + "The Marginalian": "https://www.themarginalian.org/feed/", + "Nautilus": "https://nautil.us/feed/", + "Wired": "https://www.wired.com/feed/rss", + "The Conversation": "https://theconversation.com/us/articles.atom", + "Longreads": "https://longreads.com/feed/", + "Literary Hub": "https://lithub.com/feed/", + "Atlas Obscura": "https://www.atlasobscura.com/feeds/latest", +} + +# ─── POETRY / LITERATURE ───────────────────────────────── +# Public domain via Project Gutenberg +POETRY_SOURCES = { + "Whitman": "https://www.gutenberg.org/cache/epub/1322/pg1322.txt", + "Dickinson": "https://www.gutenberg.org/cache/epub/12242/pg12242.txt", + "Whitman II": "https://www.gutenberg.org/cache/epub/8388/pg8388.txt", + "Rilke": "https://www.gutenberg.org/cache/epub/38594/pg38594.txt", + "Pound": "https://www.gutenberg.org/cache/epub/41162/pg41162.txt", + "Pound II": "https://www.gutenberg.org/cache/epub/51992/pg51992.txt", + "Eliot": "https://www.gutenberg.org/cache/epub/1567/pg1567.txt", + "Yeats": "https://www.gutenberg.org/cache/epub/38877/pg38877.txt", + "Masters": "https://www.gutenberg.org/cache/epub/1280/pg1280.txt", + "Baudelaire": "https://www.gutenberg.org/cache/epub/36098/pg36098.txt", + "Crane": "https://www.gutenberg.org/cache/epub/40786/pg40786.txt", + "Poe": "https://www.gutenberg.org/cache/epub/10031/pg10031.txt", +} + +# ─── SOURCE → LANGUAGE MAPPING ─────────────────────────── +# Headlines from these outlets render in their cultural home language +SOURCE_LANGS = { + "Der Spiegel": "de", + "DW": "de", + "France24": "fr", + "Japan Times": "ja", + "The Hindu": "hi", + "SCMP": "zh-cn", + "Al Jazeera": "ar", +} + +# ─── LOCATION → LANGUAGE ───────────────────────────────── +LOCATION_LANGS = { + r'\b(?:china|chinese|beijing|shanghai|hong kong|xi jinping)\b': 'zh-cn', + r'\b(?:japan|japanese|tokyo|osaka|kishida)\b': 'ja', + r'\b(?:korea|korean|seoul|pyongyang)\b': 'ko', + r'\b(?:russia|russian|moscow|kremlin|putin)\b': 'ru', + r'\b(?:saudi|dubai|qatar|egypt|cairo|arabic)\b': 'ar', + r'\b(?:india|indian|delhi|mumbai|modi)\b': 'hi', + r'\b(?:germany|german|berlin|munich|scholz)\b': 'de', + r'\b(?:france|french|paris|lyon|macron)\b': 'fr', + r'\b(?:spain|spanish|madrid)\b': 'es', + r'\b(?:italy|italian|rome|milan|meloni)\b': 'it', + r'\b(?:portugal|portuguese|lisbon)\b': 'pt', + r'\b(?:brazil|brazilian|são paulo|lula)\b': 'pt', + r'\b(?:greece|greek|athens)\b': 'el', + r'\b(?:turkey|turkish|istanbul|ankara|erdogan)\b': 'tr', + r'\b(?:iran|iranian|tehran)\b': 'fa', + r'\b(?:thailand|thai|bangkok)\b': 'th', + r'\b(?:vietnam|vietnamese|hanoi)\b': 'vi', + r'\b(?:ukraine|ukrainian|kyiv|kiev|zelensky)\b': 'uk', + r'\b(?:israel|israeli|jerusalem|tel aviv|netanyahu)\b': 'he', +} + +# ─── NON-LATIN SCRIPT FONTS (macOS) ────────────────────── +SCRIPT_FONTS = { + 'zh-cn': '/System/Library/Fonts/STHeiti Medium.ttc', + 'ja': '/System/Library/Fonts/ヒラギノ角ゴシック W9.ttc', + 'ko': '/System/Library/Fonts/AppleSDGothicNeo.ttc', + 'ru': '/System/Library/Fonts/Supplemental/Arial.ttf', + 'uk': '/System/Library/Fonts/Supplemental/Arial.ttf', + 'el': '/System/Library/Fonts/Supplemental/Arial.ttf', + 'he': '/System/Library/Fonts/Supplemental/Arial.ttf', + 'ar': '/System/Library/Fonts/GeezaPro.ttc', + 'fa': '/System/Library/Fonts/GeezaPro.ttc', + 'hi': '/System/Library/Fonts/Kohinoor.ttc', + 'th': '/System/Library/Fonts/ThonburiUI.ttc', +} + +# Scripts that have no uppercase +NO_UPPER = {'zh-cn', 'ja', 'ko', 'ar', 'fa', 'hi', 'th', 'he'}