klubhaus/klubhaus-doorbell
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
David Gwilliam
2026-02-12 00:45:31 -08:00
commit 5f168f370b
3024 changed files with 804889 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
libraries/FastLED/ci/dockerfiles/DockerManager.py Normal file
View File

@@ -0,0 +1,339 @@
import os
import subprocess
import sys
from typing import Any, Dict, List, Optional
class DockerManager:
def __init__(self):
# No direct client object, will use subprocess for docker CLI commands
pass
def _run_docker_command(
self,
command: List[str],
check: bool = True,
stream_output: bool = False,
timeout: Optional[int] = None,
interrupt_pattern: Optional[str] = None,
output_file: Optional[str] = None,
) -> subprocess.CompletedProcess[str]:
full_command = ["docker"] + command
print(f"Executing Docker command: {' '.join(full_command)}")
# Set environment to prevent MSYS2/Git Bash path conversion on Windows
env = os.environ.copy()
# Set UTF-8 encoding environment variables for Windows
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONUTF8"] = "1"
# Only set MSYS_NO_PATHCONV if we're in a Git Bash/MSYS2 environment
if (
"MSYSTEM" in os.environ
or os.environ.get("TERM") == "xterm"
or "bash.exe" in os.environ.get("SHELL", "")
):
env["MSYS_NO_PATHCONV"] = "1"
if stream_output:
# Use RunningProcess for streaming output
import re
import time
from ci.util.running_process import RunningProcess
proc = RunningProcess(full_command, env=env, auto_run=True)
pattern_found = False
timeout_occurred = False
start_time = time.time()
# Open output file if specified
output_handle = None
if output_file:
try:
output_handle = open(output_file, "w", encoding="utf-8")
except Exception as e:
print(f"Warning: Could not open output file {output_file}: {e}")
try:
with proc.line_iter(timeout=None) as it:
for line in it:
print(line)
# Write to output file if specified
if output_handle:
output_handle.write(line + "\n")
output_handle.flush() # Ensure immediate write
# Check for interrupt pattern (for informational purposes only)
if interrupt_pattern and re.search(interrupt_pattern, line):
print(f"Pattern detected: {line}")
pattern_found = True
# Check timeout
if timeout and (time.time() - start_time) > timeout:
print(
f"Timeout reached ({timeout}s), terminating container..."
)
timeout_occurred = True
break
# Wait for process to complete
returncode = proc.wait()
# Handle timeout case: return success (0) for timeouts as per requirement
if timeout_occurred:
print("Process terminated due to timeout - treating as success")
final_returncode = 0
else:
# Return the actual container exit code for all other cases
final_returncode = returncode
return subprocess.CompletedProcess(
args=full_command, returncode=final_returncode, stdout="", stderr=""
)
except Exception as e:
print(f"Error during streaming: {e}")
returncode = proc.wait() if hasattr(proc, "wait") else 1
return subprocess.CompletedProcess(
args=full_command, returncode=returncode, stdout="", stderr=str(e)
)
finally:
# Close output file if it was opened
if output_handle:
output_handle.close()
else:
# Use regular subprocess for non-streaming commands
result = subprocess.run(
full_command,
capture_output=True,
text=True,
check=check,
env=env,
encoding="utf-8",
errors="replace",
)
if check and result.returncode != 0:
print(
f"Docker command failed with exit code {result.returncode}",
file=sys.stderr,
)
print(f"Stdout: {result.stdout}", file=sys.stderr)
print(f"Stderr: {result.stderr}", file=sys.stderr)
return result
def pull_image(self, image_name: str, tag: str = "latest"):
"""Pulls the specified Docker image."""
print(f"Pulling image: {image_name}:{tag}")
self._run_docker_command(["pull", f"{image_name}:{tag}"])
print(f"Image {image_name}:{tag} pulled successfully.")
def run_container(
self,
image_name: str,
command: List[str],
volumes: Optional[Dict[str, Dict[str, str]]] = None,
environment: Optional[Dict[str, str]] = None,
detach: bool = False,
name: Optional[str] = None,
) -> str: # Return container ID as string
"""
Runs a Docker container with specified command, volume mounts, and environment variables.
Returns the container ID.
"""
print(
f"Running container from image: {image_name} with command: {' '.join(command)}"
)
docker_cmd = ["run", "--rm"] # --rm to automatically remove container on exit
if detach:
docker_cmd.append("-d")
if name:
docker_cmd.extend(["--name", name])
if volumes:
for host_path, container_path_info in volumes.items():
mode = container_path_info.get("mode", "rw")
docker_cmd.extend(
["-v", f"{host_path}:{container_path_info['bind']}:{mode}"]
)
if environment:
for key, value in environment.items():
docker_cmd.extend(["-e", f"{key}={value}"])
docker_cmd.append(image_name)
docker_cmd.extend(command)
result = self._run_docker_command(docker_cmd)
container_id = result.stdout.strip()
print(f"Container {container_id} started.")
return container_id
def run_container_streaming(
self,
image_name: str,
command: List[str],
volumes: Optional[Dict[str, Dict[str, str]]] = None,
environment: Optional[Dict[str, str]] = None,
name: Optional[str] = None,
timeout: Optional[int] = None,
interrupt_pattern: Optional[str] = None,
output_file: Optional[str] = None,
) -> int:
"""
Runs a Docker container with streaming output.
Returns actual container exit code (0 for success, timeout is treated as success).
Args:
image_name: Docker image to run
command: Command to execute in container
volumes: Volume mounts
environment: Environment variables
name: Container name
timeout: Timeout in seconds (if exceeded, returns 0)
interrupt_pattern: Pattern to detect in output (informational only)
output_file: Optional file path to write output to
"""
print(
f"Running container from image: {image_name} with command: {' '.join(command)}"
)
docker_cmd = ["run", "--rm"] # --rm to automatically remove container on exit
if name:
docker_cmd.extend(["--name", name])
if volumes:
for host_path, container_path_info in volumes.items():
mode = container_path_info.get("mode", "rw")
docker_cmd.extend(
["-v", f"{host_path}:{container_path_info['bind']}:{mode}"]
)
if environment:
for key, value in environment.items():
docker_cmd.extend(["-e", f"{key}={value}"])
docker_cmd.append(image_name)
docker_cmd.extend(command)
result = self._run_docker_command(
docker_cmd,
check=False,
stream_output=True,
timeout=timeout,
interrupt_pattern=interrupt_pattern,
output_file=output_file,
)
return result.returncode
def get_container_logs(self, container_id_or_name: str) -> str:
"""Retrieves logs from a container."""
print(f"Getting logs for container: {container_id_or_name}")
try:
result = self._run_docker_command(
["logs", container_id_or_name], check=False
)
if result.returncode == 0:
logs = result.stdout
print(f"Logs retrieved for {container_id_or_name}.")
return logs
else:
# Container may have exited, try to get logs anyway
print(f"Warning: docker logs returned exit code {result.returncode}")
logs = result.stdout if result.stdout else result.stderr
return logs
except Exception as e:
print(f"Warning: Failed to get container logs: {e}")
return ""
def stop_container(self, container_id_or_name: str):
"""Stops a running container."""
print(f"Stopping container: {container_id_or_name}")
self._run_docker_command(["stop", container_id_or_name])
print(f"Container {container_id_or_name} stopped.")
def remove_container(self, container_id_or_name: str):
"""Removes a container."""
print(f"Removing container: {container_id_or_name}")
self._run_docker_command(["rm", container_id_or_name])
print(f"Container {container_id_or_name} removed.")
def execute_command_in_container(
self, container_id_or_name: str, command: List[str]
) -> tuple[int, str, str]:
"""Executes a command inside a running container and returns exit code, stdout, and stderr."""
print(
f"Executing command '{' '.join(command)}' in container: {container_id_or_name}"
)
result = self._run_docker_command(
["exec", container_id_or_name] + command, check=False
)
stdout = result.stdout
stderr = result.stderr
print(
f"Command executed in {container_id_or_name}. Exit code: {result.returncode}"
)
return result.returncode, stdout, stderr
if __name__ == "__main__":
# Example Usage:
manager = DockerManager()
image_to_use = "mluis/qemu-esp32"
container_name = "test_esp32_qemu"
try:
print(f"--- Starting DockerManager test with image: {image_to_use} ---")
manager.pull_image(image_to_use)
# Example: Run a simple command in the container
print("\n--- Running a simple command ---")
container_id = manager.run_container(
image_name=image_to_use,
command=["echo", "Hello from Docker!"],
detach=True,
name=container_name,
)
# Wait a bit for the command to execute and logs to be generated
import time
time.sleep(2)
logs = manager.get_container_logs(container_id)
print("Container Logs:\n", logs)
# Check if the expected output is in the logs
if "Hello from Docker!" in logs:
print("SUCCESS: Expected output found in container logs.")
else:
print("FAILURE: Expected output NOT found in container logs.")
# Example: Execute a command inside a running container
print("\n--- Executing command inside container ---")
exit_code, stdout, stderr = manager.execute_command_in_container(
container_id, ["sh", "-c", "echo 'Executed inside!'; exit 42"]
)
print(f"Exec command stdout: {stdout}")
print(f"Exec command stderr: {stderr}")
print(f"Exec command exit code: {exit_code}")
if exit_code == 42 and "Executed inside!" in stdout:
print("SUCCESS: Command executed inside container as expected.")
else:
print("FAILURE: Command execution inside container failed.")
except subprocess.CalledProcessError as e:
print(f"Error during Docker operation: {e}", file=sys.stderr)
print(f"Stdout: {e.stdout}", file=sys.stderr)
print(f"Stderr: {e.stderr}", file=sys.stderr)
sys.exit(1)
except Exception as e:
print(f"An unexpected error occurred: {e}", file=sys.stderr)
sys.exit(1)
finally:
# Ensure cleanup
try:
manager.stop_container(container_name)
manager.remove_container(container_name)
print(f"Container {container_name} stopped and removed.")
except subprocess.CalledProcessError:
print(
f"Container {container_name} not found or already removed during cleanup."
)
except Exception as e:
print(f"Error during cleanup of {container_name}: {e}", file=sys.stderr)

106
libraries/FastLED/ci/dockerfiles/Dockerfile.qemu-esp32 Normal file
View File

@@ -0,0 +1,106 @@
# Dockerfile for ESP32 QEMU emulation environment
# Based on ESP-IDF with QEMU support
FROM espressif/idf:v5.1.2
# Install additional dependencies for QEMU
RUN apt-get update && apt-get install -y \
qemu-system-misc \
qemu-utils \
python3-pip \
&& rm -rf /var/lib/apt/lists/*
# Install Python dependencies for ESP tools
RUN pip3 install --no-cache-dir \
esptool \
pyserial
# Create workspace directory
WORKDIR /workspace
# Set up environment for QEMU ESP32
ENV QEMU_ESP32_VERSION=esp-develop-8.2.0-20240122
# Download and install ESP32 QEMU if not using system QEMU
RUN mkdir -p /opt/qemu-esp32 && \
cd /opt/qemu-esp32 && \
wget -q https://github.com/espressif/qemu/releases/download/${QEMU_ESP32_VERSION}/qemu-xtensa-softmmu-esp_develop_8.2.0_20240122-x86_64-linux-gnu.tar.xz && \
tar -xf qemu-xtensa-softmmu-esp_develop_8.2.0_20240122-x86_64-linux-gnu.tar.xz && \
rm qemu-xtensa-softmmu-esp_develop_8.2.0_20240122-x86_64-linux-gnu.tar.xz && \
ln -sf /opt/qemu-esp32/qemu/bin/qemu-system-xtensa /usr/local/bin/qemu-system-xtensa
# Download ESP32 ROM files for QEMU
RUN mkdir -p /opt/esp-rom && \
cd /opt/esp-rom && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32-v3-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32-v3-rom-eco3.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32c3-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32s2-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32s3-rom.bin
# Set environment variables for ROM files
ENV ESP_ROM_PATH=/opt/esp-rom
ENV ESP32_ROM=${ESP_ROM_PATH}/esp32-v3-rom.bin
ENV ESP32C3_ROM=${ESP_ROM_PATH}/esp32c3-rom.bin
ENV ESP32S2_ROM=${ESP_ROM_PATH}/esp32s2-rom.bin
ENV ESP32S3_ROM=${ESP_ROM_PATH}/esp32s3-rom.bin
# Create helper script for running QEMU with proper settings
RUN cat > /usr/local/bin/run-qemu-esp32 << 'EOF'
#!/bin/bash
# Helper script to run QEMU with ESP32 firmware
FIRMWARE_PATH=${1:-/workspace/firmware.bin}
MACHINE=${2:-esp32}
FLASH_SIZE=${3:-4}
# Select ROM file based on machine type
case "$MACHINE" in
esp32)
ROM_FILE=$ESP32_ROM
;;
esp32c3)
ROM_FILE=$ESP32C3_ROM
;;
esp32s2)
ROM_FILE=$ESP32S2_ROM
;;
esp32s3)
ROM_FILE=$ESP32S3_ROM
;;
*)
echo "Unknown machine type: $MACHINE"
exit 1
;;
esac
# Create flash image if needed
FLASH_IMAGE=/tmp/flash_${RANDOM}.bin
if [ -f "$FIRMWARE_PATH" ]; then
# Create empty flash image
dd if=/dev/zero of=$FLASH_IMAGE bs=1M count=$FLASH_SIZE 2>/dev/null
dd if=$FIRMWARE_PATH of=$FLASH_IMAGE bs=1 seek=$((0x10000)) conv=notrunc 2>/dev/null
else
echo "Firmware not found: $FIRMWARE_PATH"
exit 1
fi
# Run QEMU
exec qemu-system-xtensa \
-nographic \
-machine $MACHINE \
-bios $ROM_FILE \
-drive file=$FLASH_IMAGE,if=mtd,format=raw \
-global driver=timer.esp32.timg,property=wdt_disable,value=true \
"$@"
EOF
RUN chmod +x /usr/local/bin/run-qemu-esp32
# Default command
CMD ["/usr/local/bin/run-qemu-esp32"]
# Labels
LABEL maintainer="FastLED Team"
LABEL description="ESP32 QEMU emulation environment for FastLED testing"
LABEL version="1.0.0"

92
libraries/FastLED/ci/dockerfiles/Dockerfile.qemu-esp32-lite Normal file
View File

@@ -0,0 +1,92 @@
# Lightweight Dockerfile for ESP32 QEMU emulation
# Uses pre-built QEMU binaries without full ESP-IDF
FROM ubuntu:22.04
# Install minimal dependencies
RUN apt-get update && apt-get install -y \
wget \
xz-utils \
python3 \
python3-pip \
&& rm -rf /var/lib/apt/lists/*
# Install esptool for flash image creation
RUN pip3 install --no-cache-dir esptool
# Create workspace
WORKDIR /workspace
# Download and install ESP32 QEMU binaries
ENV QEMU_VERSION=esp-develop-8.2.0-20240122
RUN mkdir -p /opt/qemu && \
cd /opt/qemu && \
wget -q https://github.com/espressif/qemu/releases/download/${QEMU_VERSION}/qemu-xtensa-softmmu-${QEMU_VERSION}-x86_64-linux-gnu.tar.xz && \
tar -xf qemu-xtensa-softmmu-${QEMU_VERSION}-x86_64-linux-gnu.tar.xz && \
rm qemu-xtensa-softmmu-${QEMU_VERSION}-x86_64-linux-gnu.tar.xz && \
mv qemu /usr/local/
# Add QEMU to PATH
ENV PATH=/usr/local/qemu/bin:$PATH
# Download ESP32 ROM files
RUN mkdir -p /opt/esp-rom && \
cd /opt/esp-rom && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32-v3-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32c3-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32s2-rom.bin && \
wget -q https://github.com/espressif/qemu/raw/esp-develop/pc-bios/esp32s3-rom.bin
# Set ROM path environment variable
ENV ESP_ROM_PATH=/opt/esp-rom
# Create entrypoint script
RUN cat > /entrypoint.sh << 'EOF'
#!/bin/bash
set -e
# Default values
FIRMWARE=${FIRMWARE:-/workspace/firmware.bin}
MACHINE=${MACHINE:-esp32}
FLASH_SIZE=${FLASH_SIZE:-4}
TIMEOUT=${TIMEOUT:-30}
# Select ROM based on machine type
case "$MACHINE" in
esp32) ROM_FILE=/opt/esp-rom/esp32-v3-rom.bin ;;
esp32c3) ROM_FILE=/opt/esp-rom/esp32c3-rom.bin ;;
esp32s2) ROM_FILE=/opt/esp-rom/esp32s2-rom.bin ;;
esp32s3) ROM_FILE=/opt/esp-rom/esp32s3-rom.bin ;;
*) echo "Unknown machine: $MACHINE"; exit 1 ;;
esac
# Check if firmware exists
if [ ! -f "$FIRMWARE" ]; then
echo "ERROR: Firmware not found: $FIRMWARE"
exit 1
fi
# Create flash image
FLASH_IMG=/tmp/flash.bin
dd if=/dev/zero of=$FLASH_IMG bs=1M count=$FLASH_SIZE 2>/dev/null
dd if=$FIRMWARE of=$FLASH_IMG bs=1 seek=$((0x10000)) conv=notrunc 2>/dev/null
# Run QEMU with timeout
timeout $TIMEOUT qemu-system-xtensa \
-nographic \
-machine $MACHINE \
-bios $ROM_FILE \
-drive file=$FLASH_IMG,if=mtd,format=raw \
-global driver=timer.esp32.timg,property=wdt_disable,value=true \
"$@"
EOF
RUN chmod +x /entrypoint.sh
# Set entrypoint
ENTRYPOINT ["/entrypoint.sh"]
# Labels
LABEL maintainer="FastLED Team"
LABEL description="Lightweight ESP32 QEMU Docker image"
LABEL version="1.0.0"

205
libraries/FastLED/ci/dockerfiles/README.md Normal file
View File

@@ -0,0 +1,205 @@
# Docker-based QEMU ESP32 Testing
This directory contains Docker-based QEMU ESP32 emulation tools for FastLED testing.
## Overview
The Docker QEMU solution provides:
- Consistent testing environment across different platforms
- No need for local QEMU installation
- Isolated execution environment
- Support for multiple ESP32 variants (ESP32, ESP32-C3, ESP32-S2, ESP32-S3)
## Components
### Core Files
- `DockerManager.py` - Docker container management utilities
- `qemu_esp32_docker.py` - Main Docker QEMU runner script
- `qemu_test_integration.py` - Integration with existing test framework
- `Dockerfile.qemu-esp32` - Full ESP-IDF + QEMU environment
- `Dockerfile.qemu-esp32-lite` - Lightweight QEMU-only environment
- `docker-compose.yml` - Docker Compose configuration
## Usage
### Quick Start
1. **Run a firmware test with Docker:**
```bash
uv run ci/docker/qemu_esp32_docker.py path/to/firmware.bin
```
2. **Run with specific timeout and interrupt pattern:**
```bash
uv run ci/docker/qemu_esp32_docker.py firmware.bin \
--timeout 60 \
--interrupt-regex "Test passed"
```
3. **Use docker-compose:**
```bash
cd ci/docker
FIRMWARE_DIR=../../.pio/build/esp32dev docker-compose up qemu-esp32
```
### Integration with Test Framework
The Docker QEMU runner integrates with the existing test framework:
```bash
# Check available runners
uv run ci/docker/qemu_test_integration.py check
# Run test with automatic runner selection
uv run ci/docker/qemu_test_integration.py test --firmware path/to/firmware.bin
# Force Docker runner
uv run ci/docker/qemu_test_integration.py test --firmware firmware.bin --docker
```
### Building Docker Images
```bash
# Build lightweight image
docker build -f Dockerfile.qemu-esp32-lite -t fastled/qemu-esp32:lite .
# Build full ESP-IDF image
docker build -f Dockerfile.qemu-esp32 -t fastled/qemu-esp32:full .
# Or use docker-compose
docker-compose build
```
## Docker Images
### Pre-built Images
The runner can use pre-built images:
- `espressif/qemu:esp-develop-8.2.0-20240122` (Official Espressif image)
- `mluis/qemu-esp32:latest` (Community image)
### Custom Images
Two Dockerfiles are provided:
1. **Dockerfile.qemu-esp32-lite** (Recommended)
- Minimal Ubuntu base
- QEMU binaries only
- ~200MB image size
- Fast startup
2. **Dockerfile.qemu-esp32**
- Full ESP-IDF environment
- Development tools included
- ~2GB image size
- Complete toolchain
## Features
### Automatic Fallback
The integration module automatically selects the best available runner:
1. Native QEMU (if installed)
2. Docker QEMU (if Docker available)
3. Error if neither available
### Platform Support
Works on:
- Linux (native Docker)
- macOS (Docker Desktop)
- Windows (Docker Desktop, WSL2)
### Firmware Formats
Supports:
- Direct `firmware.bin` files
- PlatformIO build directories
- ESP-IDF build directories
## Environment Variables
- `FIRMWARE_DIR` - Directory containing firmware files
- `FIRMWARE_FILE` - Specific firmware file to run
- `MACHINE` - ESP32 variant (esp32, esp32c3, esp32s2, esp32s3)
- `FLASH_SIZE` - Flash size in MB (default: 4)
- `TIMEOUT` - Execution timeout in seconds (default: 30)
## Troubleshooting
### Docker Not Found
Install Docker:
- **Windows/Mac**: Install Docker Desktop
- **Linux**: `sudo apt-get install docker.io` or equivalent
### Permission Denied
Add user to docker group:
```bash
sudo usermod -aG docker $USER
# Log out and back in
```
### Image Pull Failed
Check internet connection and Docker Hub access. The runner will automatically try alternative images if the primary fails.
### QEMU Timeout
Increase timeout value:
```bash
uv run ci/docker/qemu_esp32_docker.py firmware.bin --timeout 120
```
## CI/CD Integration
### GitHub Actions
```yaml
- name: Run QEMU Test in Docker
run: |
uv run ci/docker/qemu_esp32_docker.py \
.pio/build/esp32dev/firmware.bin \
--timeout 60 \
--interrupt-regex "Setup complete"
```
### Local CI
```bash
# Install dependencies
uv pip install -r requirements.txt
# Run tests
uv run ci/docker/qemu_test_integration.py test \
--firmware .pio/build/esp32dev/firmware.bin
```
## Development
### Adding New ESP32 Variants
1. Update Dockerfiles with new ROM files
2. Add machine type to runner scripts
3. Test with sample firmware
### Debugging
Run interactively:
```bash
uv run ci/docker/qemu_esp32_docker.py firmware.bin --interactive
```
Enable verbose output:
```bash
docker-compose up qemu-esp32-interactive
```
## Performance
- Docker adds ~2-5 seconds overhead vs native QEMU
- First run pulls image (one-time ~100MB download)
- Subsequent runs use cached image
- Consider using lightweight image for CI/CD

0
libraries/FastLED/ci/dockerfiles/__init__.py Normal file
View File

61
libraries/FastLED/ci/dockerfiles/docker-compose.yml Normal file
View File

@@ -0,0 +1,61 @@
version: '3.8'
services:
qemu-esp32:
build:
context: .
dockerfile: Dockerfile.qemu-esp32-lite
image: fastled/qemu-esp32:latest
container_name: fastled-qemu-esp32
volumes:
# Mount firmware directory
- ${FIRMWARE_DIR:-./firmware}:/workspace:ro
# Mount output directory for logs
- ${OUTPUT_DIR:-./output}:/output
environment:
- FIRMWARE=${FIRMWARE_FILE:-/workspace/firmware.bin}
- MACHINE=${MACHINE:-esp32}
- FLASH_SIZE=${FLASH_SIZE:-4}
- TIMEOUT=${TIMEOUT:-30}
command: []
stdin_open: false
tty: false
networks:
- qemu-network
qemu-esp32-interactive:
build:
context: .
dockerfile: Dockerfile.qemu-esp32-lite
image: fastled/qemu-esp32:latest
container_name: fastled-qemu-esp32-interactive
volumes:
- ${FIRMWARE_DIR:-./firmware}:/workspace:ro
environment:
- FIRMWARE=${FIRMWARE_FILE:-/workspace/firmware.bin}
- MACHINE=${MACHINE:-esp32}
- FLASH_SIZE=${FLASH_SIZE:-4}
- TIMEOUT=${TIMEOUT:-300}
stdin_open: true
tty: true
networks:
- qemu-network
# Service for building the Docker image
builder:
build:
context: .
dockerfile: Dockerfile.qemu-esp32-lite
image: fastled/qemu-esp32:latest
command: echo "Image built successfully"
networks:
qemu-network:
driver: bridge
# Volume definitions (optional, for persistent storage)
volumes:
firmware:
driver: local
output:
driver: local

502
libraries/FastLED/ci/dockerfiles/qemu_esp32_docker.py Normal file
View File

@@ -0,0 +1,502 @@
#!/usr/bin/env python3
"""
Docker-based QEMU ESP32 Runner
Runs ESP32 firmware in QEMU using Docker containers for better isolation
and consistency across different environments.
"""
import argparse
import os
import platform
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from ci.dockerfiles.DockerManager import DockerManager
from ci.util.running_process import RunningProcess
def get_docker_env() -> Dict[str, str]:
"""Get environment for Docker commands, handling Git Bash/MSYS2 path conversion."""
env = os.environ.copy()
# Set UTF-8 encoding environment variables for Windows
env["PYTHONIOENCODING"] = "utf-8"
env["PYTHONUTF8"] = "1"
# Only set MSYS_NO_PATHCONV if we're in a Git Bash/MSYS2 environment
if (
"MSYSTEM" in os.environ
or os.environ.get("TERM") == "xterm"
or "bash.exe" in os.environ.get("SHELL", "")
):
env["MSYS_NO_PATHCONV"] = "1"
return env
def run_subprocess_safe(
cmd: List[str], **kwargs: Any
) -> subprocess.CompletedProcess[Any]:
"""Run subprocess with safe UTF-8 handling and error replacement."""
kwargs.setdefault("encoding", "utf-8")
kwargs.setdefault("errors", "replace")
kwargs.setdefault("env", get_docker_env())
return subprocess.run(cmd, **kwargs) # type: ignore[misc]
def run_docker_command_streaming(cmd: List[str]) -> int:
"""Run docker command with streaming output using RunningProcess."""
print(f"Executing: {' '.join(cmd)}")
proc = RunningProcess(cmd, env=get_docker_env(), auto_run=True)
# Stream all output to stdout
with proc.line_iter(timeout=None) as it:
for line in it:
print(line)
returncode = proc.wait()
if returncode != 0:
raise subprocess.CalledProcessError(returncode, cmd)
return returncode
def run_docker_command_no_fail(cmd: List[str]) -> int:
"""Run docker command with streaming output, but don't raise exceptions on failure."""
print(f"Executing: {' '.join(cmd)}")
proc = RunningProcess(cmd, env=get_docker_env(), auto_run=True)
# Stream all output to stdout
with proc.line_iter(timeout=None) as it:
for line in it:
print(line)
return proc.wait()
class DockerQEMURunner:
"""Runner for ESP32 QEMU emulation using Docker containers."""
DEFAULT_IMAGE = "mluis/qemu-esp32:latest"
ALTERNATIVE_IMAGE = "espressif/idf:latest"
FALLBACK_IMAGE = "espressif/idf:release-v5.2"
def __init__(self, docker_image: Optional[str] = None):
"""Initialize Docker QEMU runner.
Args:
docker_image: Docker image to use, defaults to espressif/qemu
"""
self.docker_manager = DockerManager()
self.docker_image = docker_image or self.DEFAULT_IMAGE
self.container_name = None
# Use Linux-style paths for all containers since we're using Ubuntu/Alpine
self.firmware_mount_path = "/workspace/firmware"
def check_docker_available(self) -> bool:
"""Check if Docker is available and running."""
try:
proc = RunningProcess(
["docker", "version"], env=get_docker_env(), auto_run=True
)
returncode = proc.wait(timeout=5)
return returncode == 0
except (Exception, FileNotFoundError):
return False
def pull_image(self):
"""Pull the Docker image if not already available."""
print(f"Ensuring Docker image {self.docker_image} is available...")
try:
# Check if image exists locally
proc = RunningProcess(
["docker", "images", "-q", self.docker_image],
env=get_docker_env(),
auto_run=True,
)
stdout_lines: List[str] = []
with proc.line_iter(timeout=None) as it:
for line in it:
stdout_lines.append(line)
result_stdout = "\n".join(stdout_lines)
if not result_stdout.strip():
# Image doesn't exist, pull it directly using docker command
print(f"Pulling Docker image: {self.docker_image}")
run_docker_command_streaming(["docker", "pull", self.docker_image])
print(f"Successfully pulled {self.docker_image}")
else:
print(f"Image {self.docker_image} already available locally")
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to pull image {self.docker_image}: {e}")
if self.docker_image == self.DEFAULT_IMAGE:
print(f"Trying alternative image: {self.ALTERNATIVE_IMAGE}")
self.docker_image = self.ALTERNATIVE_IMAGE
try:
print(f"Pulling alternative Docker image: {self.docker_image}")
run_docker_command_streaming(["docker", "pull", self.docker_image])
print(f"Successfully pulled {self.docker_image}")
except subprocess.CalledProcessError as e2:
print(f"Failed to pull alternative image: {e2}")
print(f"Trying fallback image: {self.FALLBACK_IMAGE}")
self.docker_image = self.FALLBACK_IMAGE
try:
print(f"Pulling fallback Docker image: {self.docker_image}")
run_docker_command_streaming(
["docker", "pull", self.docker_image]
)
print(f"Successfully pulled {self.docker_image}")
except subprocess.CalledProcessError as e3:
print(f"Failed to pull fallback image: {e3}")
raise e3
else:
raise e
def prepare_firmware(self, firmware_path: Path) -> Path:
"""Prepare firmware files for mounting into Docker container.
Args:
firmware_path: Path to firmware.bin or build directory
Returns:
Path to the prepared firmware directory
"""
# Create temporary directory for firmware files
temp_dir = Path(tempfile.mkdtemp(prefix="qemu_firmware_"))
try:
if firmware_path.is_file() and firmware_path.suffix == ".bin":
# Copy single firmware file
shutil.copy2(firmware_path, temp_dir / "firmware.bin")
# Create proper 4MB flash image for QEMU
self._create_flash_image(firmware_path, temp_dir / "flash.bin")
else:
# Copy entire build directory
if firmware_path.is_dir():
# Copy relevant files from build directory
patterns = ["*.bin", "*.elf", "partitions.csv", "flash_args"]
for pattern in patterns:
for file in firmware_path.glob(pattern):
shutil.copy2(file, temp_dir)
# Also check for PlatformIO build structure
pio_build = firmware_path / ".pio" / "build" / "esp32dev"
if pio_build.exists():
for pattern in patterns:
for file in pio_build.glob(pattern):
shutil.copy2(file, temp_dir)
# Create proper flash.bin file from firmware.bin
firmware_bin = temp_dir / "firmware.bin"
if firmware_bin.exists():
# Create proper 4MB flash image for QEMU
self._create_flash_image(firmware_bin, temp_dir / "flash.bin")
else:
# Create a minimal 4MB flash.bin for testing
flash_bin = temp_dir / "flash.bin"
flash_bin.write_bytes(
b"\xff" * (4 * 1024 * 1024)
) # 4MB of 0xFF
else:
raise ValueError(f"Invalid firmware path: {firmware_path}")
return temp_dir
except Exception as e:
# Clean up on error
shutil.rmtree(temp_dir, ignore_errors=True)
raise e
def _create_flash_image(
self, firmware_path: Path, output_path: Path, flash_size_mb: int = 4
):
"""Create a proper flash image for QEMU ESP32.
Args:
firmware_path: Path to the firmware.bin file
output_path: Path where to write the flash.bin
flash_size_mb: Flash size in MB (must be 2, 4, 8, or 16)
"""
if flash_size_mb not in [2, 4, 8, 16]:
raise ValueError(
f"Flash size must be 2, 4, 8, or 16 MB, got {flash_size_mb}"
)
flash_size = flash_size_mb * 1024 * 1024
# Read firmware content
firmware_data = firmware_path.read_bytes()
# Create flash image: firmware at beginning, rest filled with 0xFF
flash_data = firmware_data + b"\xff" * (flash_size - len(firmware_data))
# Ensure we have exactly the right size
if len(flash_data) > flash_size:
raise ValueError(
f"Firmware size ({len(firmware_data)} bytes) exceeds flash size ({flash_size} bytes)"
)
flash_data = flash_data[:flash_size] # Truncate to exact size
output_path.write_bytes(flash_data)
def build_qemu_command(
self,
firmware_name: str = "firmware.bin",
flash_size: int = 4,
machine: str = "esp32",
) -> List[str]:
"""Build QEMU command to run inside Docker container.
Args:
firmware_name: Name of firmware file in mounted directory
flash_size: Flash size in MB
machine: QEMU machine type (esp32, esp32c3, esp32s3, etc.)
Returns:
List of command arguments for QEMU
"""
# For Linux containers, use a workaround to run QEMU through Docker Desktop's Windows integration
firmware_path = f"{self.firmware_mount_path}/flash.bin"
# Determine QEMU system and machine based on target
if machine == "esp32c3":
qemu_system = "/usr/local/bin/qemu-system-riscv32"
qemu_machine = "esp32c3"
echo_target = "ESP32C3"
elif machine == "esp32s3":
qemu_system = "/usr/local/bin/qemu-system-xtensa"
qemu_machine = "esp32s3"
echo_target = "ESP32S3"
else:
# Default to ESP32 (Xtensa)
qemu_system = "/usr/local/bin/qemu-system-xtensa"
qemu_machine = "esp32"
echo_target = "ESP32"
# Use container's QEMU with proper configuration
wrapper_script = f'''#!/bin/bash
set -e
echo "Starting {echo_target} QEMU emulation..."
echo "Firmware: {firmware_path}"
echo "Machine: {qemu_machine}"
echo "QEMU system: {qemu_system}"
echo "Container: $(cat /etc/os-release | head -1)"
# Check if firmware file exists
if [ ! -f "{firmware_path}" ]; then
echo "ERROR: Firmware file not found: {firmware_path}"
exit 1
fi
# Check firmware size
FIRMWARE_SIZE=$(stat -c%s "{firmware_path}")
echo "Firmware size: $FIRMWARE_SIZE bytes"
# Copy firmware to writable location since QEMU needs write access
cp "{firmware_path}" /tmp/flash.bin
echo "Copied firmware to writable location: /tmp/flash.bin"
# Try different QEMU configurations depending on machine type
if [ "{qemu_machine}" = "esp32c3" ]; then
# ESP32C3 uses RISC-V architecture
echo "Running {qemu_system} for {qemu_machine}"
{qemu_system} \\
-nographic \\
-machine {qemu_machine} \\
-drive file="/tmp/flash.bin",if=mtd,format=raw \\
-monitor none \\
-serial mon:stdio
else
# ESP32 uses Xtensa architecture
echo "Running {qemu_system} for {qemu_machine}"
{qemu_system} \\
-nographic \\
-machine {qemu_machine} \\
-drive file="/tmp/flash.bin",if=mtd,format=raw \\
-global driver=timer.esp32.timg,property=wdt_disable,value=true \\
-monitor none \\
-serial mon:stdio
fi
echo "QEMU execution completed"
exit 0
'''
cmd = ["bash", "-c", wrapper_script]
return cmd
def run(
self,
firmware_path: Path,
timeout: int = 30,
flash_size: int = 4,
interrupt_regex: Optional[str] = None,
interactive: bool = False,
output_file: Optional[str] = None,
machine: str = "esp32",
) -> int:
"""Run ESP32/ESP32C3/ESP32S3 firmware in QEMU using Docker.
Args:
firmware_path: Path to firmware.bin or build directory
timeout: Timeout in seconds (timeout is treated as success)
flash_size: Flash size in MB
interrupt_regex: Regex pattern to detect in output (informational only)
interactive: Run in interactive mode (attach to container)
output_file: Optional file path to write QEMU output to
machine: QEMU machine type (esp32, esp32c3, esp32s3, etc.)
Returns:
Exit code: actual QEMU/container exit code, except timeout returns 0
"""
if not self.check_docker_available():
print("ERROR: Docker is not available or not running", file=sys.stderr)
print("Please install Docker and ensure it's running", file=sys.stderr)
return 1
# Pull image if needed
self.pull_image()
# Prepare firmware files
print(f"Preparing firmware from: {firmware_path}")
temp_firmware_dir = None
try:
temp_firmware_dir = self.prepare_firmware(firmware_path)
# Generate unique container name
self.container_name = f"qemu_esp32_{int(time.time())}"
# Convert Windows paths to Docker-compatible paths
def windows_to_docker_path(path_str: Union[str, Path]) -> str:
"""Convert Windows path to Docker volume mount format."""
import os
# Check if we're in Git Bash/MSYS2 environment
is_git_bash = (
"MSYSTEM" in os.environ
or os.environ.get("TERM") == "xterm"
or "bash.exe" in os.environ.get("SHELL", "")
)
if os.name == "nt" and is_git_bash:
# Convert C:\path\to\dir to /c/path/to/dir for Git Bash
path_str = str(path_str).replace("\\", "/")
if len(path_str) > 2 and path_str[1:3] == ":/": # Drive letter
path_str = "/" + path_str[0].lower() + path_str[2:]
else:
# For cmd.exe on Windows, keep native Windows paths
path_str = str(path_str)
return path_str
docker_firmware_path = windows_to_docker_path(temp_firmware_dir)
# Only mount the firmware directory - QEMU is built into the container
volumes = {
docker_firmware_path: {
"bind": self.firmware_mount_path,
"mode": "ro", # Read-only mount
},
}
# Build QEMU command
qemu_cmd = self.build_qemu_command(
firmware_name="firmware.bin", flash_size=flash_size, machine=machine
)
print(f"Running QEMU in Docker container: {self.container_name}")
print(f"QEMU command: {' '.join(qemu_cmd)}")
if interactive:
# Run interactively with streaming
return self.docker_manager.run_container_streaming(
image_name=self.docker_image,
command=qemu_cmd,
volumes=volumes,
name=self.container_name,
timeout=timeout,
interrupt_pattern=interrupt_regex,
output_file=output_file,
)
else:
# Run with streaming output and return actual exit code
return self.docker_manager.run_container_streaming(
image_name=self.docker_image,
command=qemu_cmd,
volumes=volumes,
name=self.container_name,
timeout=timeout,
interrupt_pattern=interrupt_regex,
output_file=output_file,
)
except Exception as e:
print(f"ERROR: {e}", file=sys.stderr)
return 1
finally:
# Cleanup
if temp_firmware_dir and temp_firmware_dir.exists():
shutil.rmtree(temp_firmware_dir, ignore_errors=True)
# Ensure container is stopped and removed
if self.container_name:
try:
# Stop and remove container
run_docker_command_no_fail(["docker", "stop", self.container_name])
run_docker_command_no_fail(["docker", "rm", self.container_name])
except:
pass
def main():
parser = argparse.ArgumentParser(
description="Run ESP32 firmware in QEMU using Docker"
)
parser.add_argument(
"firmware_path", type=Path, help="Path to firmware.bin file or build directory"
)
parser.add_argument(
"--docker-image",
type=str,
help=f"Docker image to use (default: {DockerQEMURunner.DEFAULT_IMAGE})",
)
parser.add_argument(
"--flash-size", type=int, default=4, help="Flash size in MB (default: 4)"
)
parser.add_argument(
"--timeout", type=int, default=30, help="Timeout in seconds (default: 30)"
)
parser.add_argument(
"--interrupt-regex", type=str, help="Regex pattern to interrupt execution"
)
parser.add_argument(
"--interactive", action="store_true", help="Run in interactive mode"
)
parser.add_argument("--output-file", type=str, help="File to write QEMU output to")
args = parser.parse_args()
if not args.firmware_path.exists():
print(
f"ERROR: Firmware path does not exist: {args.firmware_path}",
file=sys.stderr,
)
return 1
runner = DockerQEMURunner(docker_image=args.docker_image)
return runner.run(
firmware_path=args.firmware_path,
timeout=args.timeout,
flash_size=args.flash_size,
interrupt_regex=args.interrupt_regex,
interactive=args.interactive,
output_file=args.output_file,
)
if __name__ == "__main__":
sys.exit(main())

283
libraries/FastLED/ci/dockerfiles/qemu_test_integration.py Normal file
View File

@@ -0,0 +1,283 @@
#!/usr/bin/env python3
"""
Integration module for Docker-based QEMU ESP32 testing.
This module provides a bridge between the existing test infrastructure
and the Docker-based QEMU runner.
"""
import os
import platform
import shutil
import subprocess
import sys
from pathlib import Path
from typing import Optional, Union
from ci.dockerfiles.qemu_esp32_docker import DockerQEMURunner
class QEMUTestIntegration:
"""Integration class for QEMU testing with Docker fallback."""
def __init__(self, prefer_docker: bool = False):
"""Initialize QEMU test integration.
Args:
prefer_docker: If True, prefer Docker even if native QEMU is available
"""
self.prefer_docker = prefer_docker
self.docker_available = self._check_docker()
self.native_qemu_available = self._check_native_qemu()
def _check_docker(self) -> bool:
"""Check if Docker is available."""
try:
result = subprocess.run(
["docker", "version"], capture_output=True, timeout=5
)
return result.returncode == 0
except (subprocess.SubprocessError, FileNotFoundError):
return False
def _check_native_qemu(self) -> bool:
"""Check if native QEMU ESP32 is available."""
try:
# Import the native QEMU module to check
from ci.qemu_esp32 import find_qemu_binary # type: ignore[import-untyped]
return find_qemu_binary() is not None
except ImportError:
return False
def select_runner(self) -> str:
"""Select the best available runner.
Returns:
'docker' or 'native' based on availability and preference
"""
if self.prefer_docker and self.docker_available:
return "docker"
elif self.native_qemu_available:
return "native"
elif self.docker_available:
return "docker"
else:
return "none"
def run_qemu_test(
self,
firmware_path: Union[str, Path],
timeout: int = 30,
interrupt_regex: Optional[str] = None,
flash_size: int = 4,
force_runner: Optional[str] = None,
) -> int:
"""Run QEMU test with automatic runner selection.
Args:
firmware_path: Path to firmware or build directory
timeout: Test timeout in seconds
interrupt_regex: Pattern to interrupt on success
flash_size: Flash size in MB
force_runner: Force specific runner ('docker' or 'native')
Returns:
Exit code (0 for success)
"""
firmware_path = Path(firmware_path)
# Determine which runner to use
if force_runner:
runner_type = force_runner
else:
runner_type = self.select_runner()
print(f"Selected QEMU runner: {runner_type}")
if runner_type == "docker":
return self._run_docker_qemu(
firmware_path, timeout, interrupt_regex, flash_size
)
elif runner_type == "native":
return self._run_native_qemu(
firmware_path, timeout, interrupt_regex, flash_size
)
else:
print("ERROR: No QEMU runner available!", file=sys.stderr)
print("Install Docker or native QEMU ESP32", file=sys.stderr)
return 1
def _run_docker_qemu(
self,
firmware_path: Path,
timeout: int,
interrupt_regex: Optional[str],
flash_size: int,
) -> int:
"""Run QEMU test using Docker."""
print("Running QEMU test in Docker container...")
runner = DockerQEMURunner()
return runner.run(
firmware_path=firmware_path,
timeout=timeout,
interrupt_regex=interrupt_regex,
flash_size=flash_size,
)
def _run_native_qemu(
self,
firmware_path: Path,
timeout: int,
interrupt_regex: Optional[str],
flash_size: int,
) -> int:
"""Run QEMU test using native installation."""
print("Running QEMU test with native installation...")
# Import and use the native runner
from ci.qemu_esp32 import QEMURunner # type: ignore[import-untyped]
runner = QEMURunner() # type: ignore[no-untyped-call]
return runner.run( # type: ignore[no-untyped-call]
firmware_path=firmware_path,
timeout=timeout,
interrupt_regex=interrupt_regex,
flash_size=flash_size,
)
def install_qemu(self, use_docker: bool = False) -> bool:
"""Install QEMU (native or pull Docker image).
Args:
use_docker: If True, pull Docker image instead of native install
Returns:
True if installation successful
"""
if use_docker:
print("Pulling Docker QEMU image...")
try:
runner = DockerQEMURunner()
runner.pull_image()
return True
except Exception as e:
print(f"Failed to pull Docker image: {e}", file=sys.stderr)
return False
else:
print("Installing native QEMU...")
try:
# Run the native install script
result = subprocess.run(
[sys.executable, "ci/install-qemu.py"],
cwd=Path(__file__).parent.parent.parent,
)
return result.returncode == 0
except Exception as e:
print(f"Failed to install native QEMU: {e}", file=sys.stderr)
return False
def integrate_with_test_framework():
"""Integrate Docker QEMU with the existing test framework.
This function modifies the test.py to add Docker support.
"""
test_file = Path(__file__).parent.parent.parent / "test.py"
if not test_file.exists():
print(f"ERROR: test.py not found at {test_file}", file=sys.stderr)
return False
print(f"Integrating Docker QEMU support into {test_file}")
# Read the test.py file
content = test_file.read_text()
# Check if already integrated
if "docker.qemu_test_integration" in content:
print("Docker QEMU support already integrated")
return True
# Find the QEMU test section and add Docker support
integration_code = """
# Docker QEMU integration
try:
from ci.dockerfiles.qemu_test_integration import QEMUTestIntegration
DOCKER_QEMU_AVAILABLE = True
except ImportError:
DOCKER_QEMU_AVAILABLE = False
"""
# Add the import at the top of the file after other imports
import_marker = "from ci.qemu_esp32 import"
if import_marker in content:
# Add after the existing QEMU import
content = content.replace(
import_marker, integration_code + "\n" + import_marker
)
# Save the modified file
test_file.write_text(content)
print("Successfully integrated Docker QEMU support")
return True
else:
print("WARNING: Could not find appropriate location to add integration")
return False
def main():
"""Main function for testing the integration."""
import argparse
parser = argparse.ArgumentParser(
description="QEMU Test Integration with Docker support"
)
parser.add_argument(
"command",
choices=["test", "install", "integrate", "check"],
help="Command to execute",
)
parser.add_argument("--firmware", type=Path, help="Firmware path for test command")
parser.add_argument(
"--docker", action="store_true", help="Prefer Docker over native QEMU"
)
parser.add_argument(
"--timeout", type=int, default=30, help="Test timeout in seconds"
)
args = parser.parse_args()
integration = QEMUTestIntegration(prefer_docker=args.docker)
if args.command == "check":
print(f"Docker available: {integration.docker_available}")
print(f"Native QEMU available: {integration.native_qemu_available}")
print(f"Selected runner: {integration.select_runner()}")
return 0
elif args.command == "install":
success = integration.install_qemu(use_docker=args.docker)
return 0 if success else 1
elif args.command == "integrate":
success = integrate_with_test_framework()
return 0 if success else 1
elif args.command == "test":
if not args.firmware:
print("ERROR: --firmware required for test command", file=sys.stderr)
return 1
return integration.run_qemu_test(
firmware_path=args.firmware,
timeout=args.timeout,
force_runner="docker" if args.docker else None,
)
return 0
if __name__ == "__main__":
sys.exit(main())