klubhaus/klubhaus-doorbell
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
David Gwilliam
2026-02-12 00:45:31 -08:00
commit 5f168f370b
3024 changed files with 804889 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

732
libraries/FastLED/ci/install_qemu_esp32.py Normal file
View File

@@ -0,0 +1,732 @@
#!/usr/bin/env python3
"""
ESP32 QEMU Installation Script
Installs ESP32-compatible QEMU emulator using ESP-IDF tools.
Cross-platform support for Linux, macOS, and Windows.
WARNING: Never use sys.stdout.flush() in this file!
It causes blocking issues on Windows that hang QEMU processes.
Python's default buffering behavior works correctly across platforms.
"""
import hashlib
import os
import platform
import shutil
import subprocess
import sys
import tarfile
import tempfile
import threading
from pathlib import Path
from typing import Dict, List, Optional, Union
from download import download # type: ignore
from ci.util.running_process import RunningProcess
def get_platform_info():
"""Get platform-specific information for QEMU installation."""
print("IMMEDIATE DEBUG: get_platform_info() called")
print("Detecting platform information...")
system = platform.system().lower()
print(f"IMMEDIATE DEBUG: platform.system() = {system}")
machine = platform.machine().lower()
print(f"IMMEDIATE DEBUG: platform.machine() = {machine}")
print(f"System: {system}, Machine: {machine}")
# Normalize machine architecture
if machine in ["x86_64", "amd64"]:
arch = "x86_64"
elif machine in ["aarch64", "arm64"]:
arch = "arm64"
elif machine.startswith("arm"):
arch = "arm"
else:
arch = machine
return system, arch
def install_system_dependencies():
"""Install system dependencies required for QEMU."""
system, _ = get_platform_info()
print("Installing system dependencies...")
if system == "linux":
# Ubuntu/Debian dependencies
deps = [
"libgcrypt20",
"libglib2.0-0",
"libpixman-1-0",
"libsdl2-2.0-0",
"libslirp0",
]
try:
# Check if we can install packages (requires sudo)
print("Updating package list...")
process = RunningProcess(["sudo", "apt-get", "update"])
result = process.wait(echo=True)
if result == 0:
print("Installing system dependencies:", " ".join(deps))
process = RunningProcess(["sudo", "apt-get", "install", "-y"] + deps)
result = process.wait(echo=True)
if result != 0:
raise subprocess.CalledProcessError(
result if result is not None else 1,
"apt-get install",
)
print("System dependencies installed")
else:
print("WARNING: Could not install system dependencies (sudo required)")
print("Please install manually:", " ".join(deps))
except subprocess.CalledProcessError:
print("WARNING: Failed to install system dependencies")
elif system == "darwin":
# macOS - dependencies usually available via system
print("macOS system dependencies should be available")
elif system == "windows":
# Windows - QEMU binaries are self-contained
print("Windows - no additional dependencies required")
def download_esp_idf_tools():
"""Download and setup ESP-IDF tools for QEMU installation."""
tools_dir = Path.home() / ".espressif"
tools_dir.mkdir(exist_ok=True)
# ESP-IDF tools script URL
tools_url = (
"https://raw.githubusercontent.com/espressif/esp-idf/master/tools/idf_tools.py"
)
tools_script = tools_dir / "idf_tools.py"
if not tools_script.exists():
print("Downloading ESP-IDF tools...")
print(f"Source: {tools_url}")
print(f"Destination: {tools_script}")
download(tools_url, tools_script)
print(f"Downloaded ESP-IDF tools to {tools_script}")
else:
print(f"ESP-IDF tools already exist at {tools_script}")
return tools_script
def check_command_available(command: str) -> bool:
"""Check if a command is available in PATH."""
print(f"Checking if {command} is available...")
try:
result = subprocess.run(
[command, "--version"], capture_output=True, text=True, timeout=10
)
available = result.returncode == 0
print(f"{command} {'found' if available else 'not found'}")
return available
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
print(f"{command} not found or timed out")
return False
def verify_file_hash(file_path: Path, expected_hash: str) -> bool:
"""Verify SHA256 hash of a downloaded file."""
print(f"Verifying SHA256 hash of {file_path}...")
try:
with open(file_path, "rb") as f:
file_hash = hashlib.sha256(f.read()).hexdigest()
if file_hash == expected_hash:
print(f"SHA256 verification passed: {file_hash}")
return True
else:
print(f"SHA256 verification failed!")
print(f"Expected: {expected_hash}")
print(f"Got: {file_hash}")
return False
except Exception as e:
print(f"Error verifying hash: {e}")
return False
def try_espressif_binary_install():
"""Download and install QEMU using Espressif official precompiled binaries."""
try:
print("\n--- Espressif Official Binary Installation ---")
print("Downloading QEMU from Espressif official releases...")
# Create local QEMU directory (project-local)
qemu_dir = Path(".cache") / "qemu"
qemu_dir.mkdir(parents=True, exist_ok=True)
# Check if already installed
qemu_xtensa_binary = qemu_dir / "qemu-system-xtensa.exe"
qemu_riscv32_binary = qemu_dir / "qemu-system-riscv32.exe"
print(f"Checking for existing QEMU at {qemu_xtensa_binary}...")
if qemu_xtensa_binary.exists() and qemu_riscv32_binary.exists():
print(
f"QEMU already installed at {qemu_xtensa_binary} and {qemu_riscv32_binary}"
)
return True
else:
print("QEMU not found, proceeding with download...")
# Espressif official QEMU binaries for Windows x64
binaries = [
{
"name": "QEMU-Xtensa (ESP32/ESP32-S2/ESP32-S3)",
"url": "https://github.com/espressif/qemu/releases/download/esp-develop-9.2.2-20250817/qemu-xtensa-softmmu-esp_develop_9.2.2_20250817-x86_64-w64-mingw32.tar.xz",
"sha256": "ef550b912726997f3c1ff4a4fb13c1569e2b692efdc5c9f9c3c926a8f7c540fa",
"binary_name": "qemu-system-xtensa.exe",
},
{
"name": "QEMU-RISCV32 (ESP32-C3/ESP32-H2/ESP32-C2)",
"url": "https://github.com/espressif/qemu/releases/download/esp-develop-9.2.2-20250817/qemu-riscv32-softmmu-esp_develop_9.2.2_20250817-x86_64-w64-mingw32.tar.xz",
"sha256": "9474015f24d27acb7516955ec932e5307226bd9d6652cdc870793ed36010ab73",
"binary_name": "qemu-system-riscv32.exe",
},
]
for binary_info in binaries:
print(f"\nDownloading {binary_info['name']}...")
print("This may take a few minutes depending on your internet connection.")
archive_path = qemu_dir / f"{binary_info['binary_name']}.tar.xz"
print(f"Downloading from: {binary_info['url']}")
print(f"Saving to: {archive_path}")
try:
# Download the archive
download(binary_info["url"], archive_path)
print(f"Download completed: {archive_path}")
# Verify SHA256 hash
if not verify_file_hash(archive_path, binary_info["sha256"]):
print(f"Hash verification failed for {binary_info['name']}")
archive_path.unlink() # Remove corrupted file
return False
# Extract the archive
print(f"Extracting {binary_info['name']}...")
with tarfile.open(archive_path, "r:xz") as tar:
# List contents to understand structure
members = tar.getnames()
print(f"Archive contains {len(members)} files")
# Extract to temporary directory first
temp_extract_dir = qemu_dir / "temp_extract"
temp_extract_dir.mkdir(exist_ok=True)
tar.extractall(temp_extract_dir)
print(f"Extracted to temporary directory: {temp_extract_dir}")
# Find the binary in the extracted contents
found_binary = None
for root, dirs, files in os.walk(temp_extract_dir):
for file in files:
if file == binary_info["binary_name"]:
found_binary = Path(root) / file
break
if found_binary:
break
if found_binary:
target_binary = qemu_dir / binary_info["binary_name"]
print(
f"Copying {binary_info['binary_name']} to {target_binary}"
)
shutil.copy2(found_binary, target_binary)
# Make sure it's executable (though Windows doesn't use execute bit)
target_binary.chmod(0o755)
print(f"Successfully installed {binary_info['binary_name']}")
else:
print(
f"ERROR: Could not find {binary_info['binary_name']} in extracted archive"
)
return False
# Clean up temporary extraction directory
shutil.rmtree(temp_extract_dir)
# Clean up downloaded archive
archive_path.unlink()
print(f"Cleaned up {archive_path}")
except Exception as e:
print(f"Error downloading/extracting {binary_info['name']}: {e}")
# Clean up on error
if archive_path.exists():
archive_path.unlink()
return False
print("Espressif binary installation completed successfully")
return True
except Exception as e:
print(f"Espressif binary installation error: {e}")
return False
def try_direct_download_install():
"""Download and install QEMU directly from the official source."""
print("\n--- Legacy Installer Method (Requires Admin) ---")
print("WARNING: This method requires administrator privileges and may fail")
print("Consider using the Espressif binary method instead")
try:
print("Downloading QEMU directly from official source...")
# Create local QEMU directory (project-local)
qemu_dir = Path(".cache") / "qemu"
qemu_dir.mkdir(parents=True, exist_ok=True)
# Check if already installed
qemu_binary = qemu_dir / "qemu-system-xtensa.exe"
print(f"Checking for existing QEMU at {qemu_binary}...")
if qemu_binary.exists():
print(f"QEMU already installed at {qemu_binary}")
return True
else:
print("QEMU not found, proceeding with download...")
# Download QEMU from official source
print("Downloading QEMU for Windows...")
print("This may take several minutes depending on your internet connection.")
# QEMU download URL from the official Windows builds
qemu_url = "https://qemu.weilnetz.de/w64/2024/qemu-w64-setup-20241220.exe"
installer_path = qemu_dir / "qemu-installer.exe"
print(f"Downloading from: {qemu_url}")
print(f"Saving to: {installer_path}")
try:
# Download the installer
download(qemu_url, installer_path)
print(f"Download completed: {installer_path}")
# Run the installer in silent mode
print("Running QEMU installer in silent mode...")
print("Note: This may require administrator privileges")
# Try different installation approaches
install_attempts = [
# Attempt 1: Install to custom directory (may require admin)
[
str(installer_path),
"/VERYSILENT", # Very silent install
"/SUPPRESSMSGBOXES", # Suppress message boxes
f"/DIR={qemu_dir.absolute()}", # Install directory
],
# Attempt 2: Install to default location (may require admin)
[
str(installer_path),
"/VERYSILENT", # Very silent install
"/SUPPRESSMSGBOXES", # Suppress message boxes
],
# Attempt 3: Use basic silent install
[
str(installer_path),
"/S", # Silent install
],
]
result = None
process = None
for i, install_cmd in enumerate(install_attempts, 1):
print(
f"Installation attempt {i}/{len(install_attempts)}: {' '.join(install_cmd)}"
)
try:
process = RunningProcess(
install_cmd,
timeout=300, # 5 minute timeout
)
result = process.wait(echo=True)
if result == 0:
print(f"Installation attempt {i} succeeded")
break
else:
print(
f"Installation attempt {i} failed with exit code: {result}"
)
if i < len(install_attempts):
print("Trying next installation method...")
except Exception as e:
print(f"Installation attempt {i} failed with error: {e}")
if i < len(install_attempts):
print("Trying next installation method...")
if result == 0:
print("QEMU installer completed successfully")
# Look for the installed qemu-system-xtensa.exe
for qemu_path in qemu_dir.rglob("qemu-system-xtensa.exe"):
if qemu_path.is_file():
print(f"Found QEMU binary at: {qemu_path}")
# If it's not in the expected location, copy it there
if qemu_path != qemu_binary:
print(
f"Copying QEMU binary to standard location: {qemu_binary}"
)
shutil.copy2(qemu_path, qemu_binary)
# Clean up installer
if installer_path.exists():
installer_path.unlink()
print("Direct download installation completed successfully")
return True
print("QEMU installer completed but qemu-system-xtensa.exe not found")
else:
print(f"QEMU installer failed with exit code: {result}")
if process:
stderr_output = process.stdout
if stderr_output:
print(f"Error output: {stderr_output[:200]}")
except Exception as download_error:
print(f"Download/install error: {download_error}")
return False
except Exception as e:
print(f"Direct download installation error: {e}")
return False
def try_windows_installation():
"""Try installing QEMU using Espressif binaries for Windows."""
print("Attempting automatic QEMU installation on Windows...")
# Try Espressif official binaries first (recommended method)
print("Using Espressif official binaries...")
if try_espressif_binary_install():
print("Espressif binary installation completed, verifying...")
# Verify installation worked
if find_qemu_binary():
print("SUCCESS: QEMU installed via Espressif official binaries")
return True
else:
print(
"WARNING: Espressif binary installation reported success but QEMU binary not found"
)
else:
print("Espressif binary installation failed")
# Fall back to legacy installer method (requires admin privileges)
print("Falling back to legacy installer method...")
if try_direct_download_install():
print("Legacy installer completed, verifying...")
# Verify installation worked
if find_qemu_binary():
print("SUCCESS: QEMU installed via legacy installer")
return True
else:
print(
"WARNING: Legacy installer reported success but QEMU binary not found"
)
else:
print("Legacy installer failed")
print("Windows QEMU installation failed")
return False
def install_qemu_esp32():
"""Install ESP32 QEMU using ESP-IDF tools."""
print("\n=== Starting ESP32 QEMU Installation ===")
system, arch = get_platform_info()
print(f"\nChecking for ESP32 QEMU for {system}-{arch}...")
# Check if already installed
print("Searching for existing QEMU installation...")
qemu_binary = find_qemu_binary()
print(f"IMMEDIATE DEBUG: find_qemu_binary returned: {qemu_binary}")
if qemu_binary:
print("ESP32 QEMU already installed")
return True
else:
print("No existing QEMU installation found")
# Try automated installation
print("Attempting automatic installation...")
if system == "linux":
# Check if we're in CI first for sudo operations
if os.getenv("CI") == "true" or os.getenv("GITHUB_ACTIONS") == "true":
try:
# Try to install QEMU from package manager in CI
print("Installing QEMU via apt-get...")
print("Updating package list...")
process = RunningProcess(["sudo", "apt-get", "update"])
result = process.wait(echo=True)
if result == 0:
print("Package list updated successfully")
print("Installing QEMU packages...")
print("Installing: qemu-system-misc qemu-system-xtensa")
process = RunningProcess(
[
"sudo",
"apt-get",
"install",
"-y",
"qemu-system-misc",
"qemu-system-xtensa",
]
)
result = process.wait(echo=True)
if result == 0:
print("QEMU packages installed successfully via apt-get")
return True
else:
print("Failed to install QEMU packages via apt-get")
print("Return code:", result)
except Exception as e:
print(f"Automatic installation failed: {e}")
else:
print(
"Local Linux environment - automatic installation requires sudo privileges"
)
elif system == "windows":
print("Windows platform detected, using direct download...")
# Try direct download installation (works in both CI and local environments)
if try_windows_installation():
print("Windows QEMU installation successful")
return True
else:
print("Windows installation failed")
elif system == "darwin":
# Try macOS installation methods
print("macOS platform detected")
print("Attempting macOS installation...")
print("Note: Homebrew installation could be added here in the future")
# Could add homebrew installation here in the future
# If not found, provide installation instructions
print("ESP32 QEMU not found. Please install using one of these methods:")
print()
if system == "windows":
print("Windows Installation Options:")
print()
print("Option 1: Espressif Official Binaries (Recommended)")
print(
" This is what the automatic installer uses - no administrator privileges required"
)
print(
" Espressif QEMU Xtensa: https://github.com/espressif/qemu/releases/download/esp-develop-9.2.2-20250817/qemu-xtensa-softmmu-esp_develop_9.2.2_20250817-x86_64-w64-mingw32.tar.xz"
)
print(
" Espressif QEMU RISC-V32: https://github.com/espressif/qemu/releases/download/esp-develop-9.2.2-20250817/qemu-riscv32-softmmu-esp_develop_9.2.2_20250817-x86_64-w64-mingw32.tar.xz"
)
print(" Extract to .cache/qemu/ directory")
print()
print("Option 2: Official QEMU Installer (Requires Administrator)")
print(
" Download QEMU from: https://qemu.weilnetz.de/w64/2024/qemu-w64-setup-20241220.exe"
)
print(" Run the installer and ensure qemu-system-xtensa.exe is in your PATH")
print()
print("Option 3: ESP-IDF (includes QEMU)")
print(
" Download ESP-IDF from: https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/windows-setup.html"
)
print()
print("Option 4: Manual Installation")
print(" Download QEMU from: https://www.qemu.org/download/")
print(" Make sure qemu-system-xtensa.exe is in your PATH")
elif system == "linux":
print("Option 1: Install via ESP-IDF")
print(" git clone --recursive https://github.com/espressif/esp-idf.git")
print(" cd esp-idf && ./install.sh esp32s3")
print(" source ./export.sh")
print()
print("Option 2: Install QEMU from package manager")
print(" sudo apt-get install qemu-system-misc qemu-system-xtensa")
elif system == "darwin":
print("Option 1: Install via ESP-IDF")
print(" git clone --recursive https://github.com/espressif/esp-idf.git")
print(" cd esp-idf && ./install.sh esp32s3")
print(" source ./export.sh")
print()
print("Option 2: Install via Homebrew")
print(" brew install qemu")
print()
print("For CI/GitHub Actions, QEMU will be cached automatically once installed.")
return False
def find_qemu_binary() -> Optional[Path]:
"""Find the installed QEMU binary."""
print("IMMEDIATE DEBUG: find_qemu_binary() called")
system, _ = get_platform_info()
print(f"IMMEDIATE DEBUG: got system: {system}")
# Look for both Xtensa and RISC-V32 binaries
binary_names = []
if system == "windows":
binary_names = ["qemu-system-xtensa.exe", "qemu-system-riscv32.exe"]
else:
binary_names = ["qemu-system-xtensa", "qemu-system-riscv32"]
print(f"IMMEDIATE DEBUG: looking for binaries: {binary_names}")
# ESP-IDF installation paths and portable installation
esp_paths = [
Path(".cache") / "qemu", # Project-local portable installation directory
Path.home()
/ ".cache"
/ "qemu", # Legacy user cache location (for backward compatibility)
Path.home() / ".espressif" / "tools" / "qemu-xtensa",
Path.home() / ".espressif" / "python_env",
Path("/opt/espressif/tools/qemu-xtensa"),
]
# Search in ESP-IDF paths first
for base_path in esp_paths:
print(f"Searching in: {base_path}")
try:
if base_path.exists():
print(f" Directory exists, searching for QEMU binaries...")
for binary_name in binary_names:
print(
f" Starting recursive search for {binary_name} in {base_path}..."
)
for qemu_path in base_path.rglob(binary_name):
print(f" Checking: {qemu_path}")
if qemu_path.is_file():
print(f"Found QEMU binary at: {qemu_path}")
return qemu_path
print(f" No QEMU binaries found in {base_path}")
else:
print(f" Directory does not exist")
except Exception as e:
print(f" Error searching {base_path}: {e}")
# Check system PATH for each binary
print("Checking system PATH...")
for binary_name in binary_names:
try:
cmd = ["where" if system == "windows" else "which", binary_name]
print(f"Running: {' '.join(cmd)}")
print("About to call subprocess.run...")
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
print("subprocess.run completed")
if result.returncode == 0:
# Take first line in case of multiple results (Windows)
first_line = result.stdout.strip().split("\n")[0]
qemu_path = Path(first_line)
print(f"Found QEMU binary in PATH: {qemu_path}")
return qemu_path
else:
print(f"Command returned code {result.returncode}")
except subprocess.TimeoutExpired:
print(f"PATH check for {binary_name} timed out after 10 seconds")
except (subprocess.SubprocessError, FileNotFoundError) as e:
print(f"Error checking PATH for {binary_name}: {e}")
print("ERROR: Could not find any QEMU binary")
return None
def create_qemu_wrapper():
"""Create a wrapper script for easy QEMU access."""
# DISABLED: This function was overwriting the actual qemu-esp32.py runner script
# The qemu-esp32.py script already exists and handles all the necessary
# functionality for running ESP32 firmware in QEMU with proper argument parsing
print("Skipping wrapper creation - using existing qemu-esp32.py runner script")
return True
def main():
"""Main installation routine."""
print("IMMEDIATE DEBUG: main() function called")
print("=== ESP32 QEMU Installation ===")
print("Step 1/5: Starting installation process...")
# Show platform info
print("Step 2/5: Detecting platform...")
system, arch = get_platform_info()
print(f"IMMEDIATE DEBUG: get_platform_info completed, got: {system}, {arch}")
print(f"Platform: {system}-{arch}")
# Install system dependencies
print("Step 3/5: Installing system dependencies...")
install_system_dependencies()
print("IMMEDIATE DEBUG: install_system_dependencies completed")
print("System dependencies step completed")
# Install ESP32 QEMU
print("Step 4/5: Installing ESP32 QEMU...")
qemu_installed = install_qemu_esp32()
print(
f"IMMEDIATE DEBUG: install_qemu_esp32 completed with result: {qemu_installed}"
)
print(f"ESP32 QEMU installation step completed (success: {qemu_installed})")
if qemu_installed:
# Create wrapper script
print("Step 5/5: Creating wrapper scripts...")
if not create_qemu_wrapper():
print("WARNING: Could not create wrapper script, but QEMU is available")
else:
print("QEMU installation required for ESP32 emulation")
sys.exit(1)
print("=== Installation Complete ===")
print("ESP32 QEMU is ready to use!")
# Test installation
print("\n=== Verifying Installation ===")
print("Searching for QEMU binary...")
qemu_binary = find_qemu_binary()
if qemu_binary:
print(f"Testing QEMU binary: {qemu_binary}")
try:
print("Running version check...")
result = subprocess.run(
[str(qemu_binary), "--version"],
capture_output=True,
text=True,
timeout=10,
)
if result.returncode == 0:
print(f"QEMU version: {result.stdout.strip()}")
print("Installation verification successful!")
sys.exit(0) # Explicit success exit
else:
print("WARNING: QEMU installed but version check failed")
print(f"Version check return code: {result.returncode}")
sys.exit(1)
except Exception as e:
print(f"WARNING: Could not verify QEMU installation: {e}")
sys.exit(1)
else:
print("ERROR: QEMU installation verification failed")
sys.exit(1)
if __name__ == "__main__":
main()