initial commit
This commit is contained in:
643
libraries/FastLED/ci/build_api.py
Normal file
643
libraries/FastLED/ci/build_api.py
Normal file
@@ -0,0 +1,643 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
FastLED Unified Build API
|
||||
|
||||
This module provides a unified build system for both unit tests and examples,
|
||||
centralizing PCH and library building while providing a simple, elegant API.
|
||||
|
||||
Key Features:
|
||||
- Unified API for both unit tests and examples
|
||||
- Automatic PCH header building and reuse
|
||||
- Automatic libfastled.a building and reuse
|
||||
- Organized build directories under .build/
|
||||
- Efficient multiple target building
|
||||
- Simple, elegant interface
|
||||
|
||||
Usage:
|
||||
# Unit test building
|
||||
unit_builder = BuildAPI(
|
||||
config_file="ci/build_unit.toml",
|
||||
build_dir=".build/unit",
|
||||
build_type=BuildType.UNIT_TEST
|
||||
)
|
||||
unit_builder.build_targets(["test_json.cpp", "test_color.cpp"])
|
||||
|
||||
# Example building
|
||||
example_builder = BuildAPI(
|
||||
config_file="ci/build_example.toml",
|
||||
build_dir=".build/examples",
|
||||
build_type=BuildType.EXAMPLE
|
||||
)
|
||||
example_builder.build_targets(["Blink.ino", "DemoReel100.ino"])
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
from ci.ci.fingerprint_cache import FingerprintCache
|
||||
from ci.compiler.clang_compiler import (
|
||||
BuildFlags,
|
||||
Compiler,
|
||||
CompilerOptions,
|
||||
LibarchiveOptions,
|
||||
)
|
||||
from ci.compiler.test_example_compilation import get_fastled_core_sources
|
||||
from ci.util.paths import PROJECT_ROOT
|
||||
from ci.util.running_process import subprocess_run
|
||||
|
||||
|
||||
class BuildType(Enum):
|
||||
"""Build type enumeration"""
|
||||
|
||||
UNIT_TEST = "unit_test"
|
||||
EXAMPLE = "example"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildTarget:
|
||||
"""Represents a single build target (test file or example)"""
|
||||
|
||||
source_file: Path
|
||||
name: str
|
||||
output_dir: Path
|
||||
|
||||
def get_output_executable(self) -> Path:
|
||||
"""Get the expected output executable path"""
|
||||
if self.source_file.suffix == ".cpp":
|
||||
# Unit test: test_name -> test_name.exe
|
||||
base_name = self.source_file.stem
|
||||
return self.output_dir / f"{base_name}.exe"
|
||||
else:
|
||||
# Example: Blink.ino -> Blink.exe
|
||||
base_name = self.source_file.stem
|
||||
return self.output_dir / f"{base_name}.exe"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildResult:
|
||||
"""Result of a build operation"""
|
||||
|
||||
target: BuildTarget
|
||||
success: bool
|
||||
message: str
|
||||
build_time: float
|
||||
|
||||
|
||||
class BuildAPI:
|
||||
"""
|
||||
Unified build API for both unit tests and examples.
|
||||
|
||||
This class centralizes all build operations including PCH building,
|
||||
library creation, and target compilation. It provides a simple,
|
||||
elegant interface while handling the complexity internally.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
build_flags_toml: str | Path,
|
||||
build_dir: str | Path,
|
||||
build_type: BuildType,
|
||||
use_pch: bool = True,
|
||||
parallel: bool = True,
|
||||
clean: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the build API.
|
||||
|
||||
Args:
|
||||
build_flags_toml: Path to build_flags.toml file (build_unit.toml or build_example.toml)
|
||||
build_dir: Directory for build outputs (e.g., .build/unit or .build/examples)
|
||||
build_type: Type of build (UNIT_TEST or EXAMPLE)
|
||||
use_pch: Whether to use precompiled headers
|
||||
parallel: Whether to enable parallel compilation
|
||||
clean: Whether to force a clean build
|
||||
"""
|
||||
self.build_flags_toml = Path(build_flags_toml)
|
||||
self.build_dir = Path(build_dir)
|
||||
self.build_type = build_type
|
||||
self.use_pch = use_pch
|
||||
self.parallel = parallel
|
||||
self.clean = clean
|
||||
|
||||
# Ensure build directory exists
|
||||
self.build_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Set up subdirectories for organization
|
||||
self.artifacts_dir = self.build_dir / "artifacts" # PCH, lib files
|
||||
self.targets_dir = self.build_dir / "targets" # Individual target builds
|
||||
self.cache_dir = self.build_dir / "cache" # Fingerprint caches
|
||||
|
||||
for directory in [self.artifacts_dir, self.targets_dir, self.cache_dir]:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Initialize internal state
|
||||
self._compiler: Optional[Compiler] = None
|
||||
self._pch_file: Optional[Path] = None
|
||||
self._library_file: Optional[Path] = None
|
||||
self._fingerprint_cache: Optional[FingerprintCache] = None
|
||||
|
||||
# Parse build flags from TOML file
|
||||
self._parse_build_flags()
|
||||
|
||||
# Create compiler from TOML configuration
|
||||
self._create_compiler()
|
||||
|
||||
def _parse_build_flags(self) -> None:
|
||||
"""Parse build flags from TOML file"""
|
||||
if not self.build_flags_toml.exists():
|
||||
raise RuntimeError(
|
||||
f"Build flags TOML file not found: {self.build_flags_toml}"
|
||||
)
|
||||
|
||||
print(f"[BUILD API] Parsing build flags from {self.build_flags_toml}")
|
||||
|
||||
# Parse build flags from TOML
|
||||
self.build_flags = BuildFlags.from_toml_file(
|
||||
self.build_flags_toml,
|
||||
quick_build=True, # Use quick build mode for faster iteration
|
||||
strict_mode=False,
|
||||
)
|
||||
|
||||
# Initialize fingerprint cache
|
||||
cache_file = self.cache_dir / "fingerprint_cache.json"
|
||||
self._fingerprint_cache = FingerprintCache(cache_file)
|
||||
|
||||
def _create_compiler(self) -> None:
|
||||
"""Create compiler from TOML configuration ONLY - no inline settings allowed"""
|
||||
from ci.compiler.clang_compiler import create_compiler_options_from_toml
|
||||
|
||||
# ALL settings must come from the TOML file - NO hardcoded/inline settings allowed
|
||||
print(f"[BUILD API] Creating compiler from TOML configuration")
|
||||
|
||||
# Set up PCH output path
|
||||
pch_output_path: Optional[str] = None
|
||||
if self.use_pch:
|
||||
pch_output_path = str(
|
||||
self.artifacts_dir / f"{self.build_type.value}_pch.hpp.pch"
|
||||
)
|
||||
|
||||
# Use the proper function to create CompilerOptions from TOML - NO inline settings
|
||||
self.compiler_settings = create_compiler_options_from_toml(
|
||||
toml_path=self.build_flags_toml,
|
||||
include_path=str(PROJECT_ROOT / "src"), # Base include path only
|
||||
quick_build=True, # Use quick build mode for faster iteration
|
||||
strict_mode=False,
|
||||
# Pass PCH and parallel settings from constructor
|
||||
use_pch=self.use_pch,
|
||||
pch_output_path=pch_output_path,
|
||||
parallel=self.parallel,
|
||||
)
|
||||
|
||||
# Create compiler instance with TOML-loaded settings
|
||||
self._compiler = Compiler(self.compiler_settings, self.build_flags)
|
||||
|
||||
def _ensure_pch_built(self) -> bool:
|
||||
"""
|
||||
Ensure PCH header is built and ready.
|
||||
|
||||
Returns:
|
||||
bool: True if PCH is ready, False otherwise
|
||||
"""
|
||||
if not self.use_pch or not self._compiler:
|
||||
return True
|
||||
|
||||
print(f"[BUILD API] Building PCH header for {self.build_type.value}")
|
||||
|
||||
# Check if we need to rebuild PCH
|
||||
if not self.clean and self._pch_file and self._pch_file.exists():
|
||||
# TODO: Add proper dependency checking here
|
||||
print(f"[BUILD API] Reusing existing PCH: {self._pch_file}")
|
||||
return True
|
||||
|
||||
# Build PCH
|
||||
success = self._compiler.create_pch_file()
|
||||
if success and self.compiler_settings.pch_output_path:
|
||||
self._pch_file = Path(self.compiler_settings.pch_output_path)
|
||||
print(f"[BUILD API] PCH built successfully: {self._pch_file}")
|
||||
else:
|
||||
print(f"[BUILD API] PCH build failed")
|
||||
|
||||
return success
|
||||
|
||||
def _ensure_library_built(self) -> bool:
|
||||
"""
|
||||
Ensure libfastled.a is built and ready.
|
||||
|
||||
Returns:
|
||||
bool: True if library is ready, False otherwise
|
||||
"""
|
||||
if not self._compiler:
|
||||
return False
|
||||
|
||||
lib_name = f"libfastled_{self.build_type.value}.a"
|
||||
self._library_file = self.artifacts_dir / lib_name
|
||||
|
||||
print(f"[BUILD API] Building FastLED library for {self.build_type.value}")
|
||||
|
||||
# Check if we need to rebuild library
|
||||
if not self.clean and self._library_file.exists():
|
||||
# TODO: Add proper dependency checking here
|
||||
print(f"[BUILD API] Reusing existing library: {self._library_file}")
|
||||
return True
|
||||
|
||||
# Prefer UNITY chunk build for faster library creation
|
||||
try:
|
||||
unity_dir = self.artifacts_dir / "unity"
|
||||
unity_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Collect FastLED core sources (consistent with example compilation)
|
||||
all_sources = [
|
||||
p for p in get_fastled_core_sources() if p.name != "stub_main.cpp"
|
||||
]
|
||||
if not all_sources:
|
||||
print("[BUILD API] No FastLED sources found for unity build")
|
||||
return False
|
||||
|
||||
# Deterministic sort relative to project root
|
||||
project_root = PROJECT_ROOT
|
||||
|
||||
def sort_key(p: Path) -> str:
|
||||
try:
|
||||
rel = p.relative_to(project_root)
|
||||
except Exception:
|
||||
rel = p
|
||||
return rel.as_posix()
|
||||
|
||||
all_sources = sorted(all_sources, key=sort_key)
|
||||
|
||||
# Partition into chunks (1..4 typical)
|
||||
total = len(all_sources)
|
||||
max_chunks = min(4, max(1, (os.cpu_count() or 2) // 2))
|
||||
chunks = max_chunks if total >= max_chunks else max(1, total)
|
||||
|
||||
# Prepare compile options (PCH off for unity)
|
||||
compile_opts = CompilerOptions(
|
||||
include_path=self._compiler.settings.include_path,
|
||||
compiler=self._compiler.settings.compiler,
|
||||
defines=self._compiler.settings.defines,
|
||||
std_version=self._compiler.settings.std_version,
|
||||
compiler_args=self._compiler.settings.compiler_args,
|
||||
use_pch=False,
|
||||
additional_flags=["-c"],
|
||||
parallel=self.parallel,
|
||||
)
|
||||
|
||||
# Compute chunk sizes
|
||||
base = total // chunks
|
||||
rem = total % chunks
|
||||
start = 0
|
||||
unity_objects: list[Path] = []
|
||||
|
||||
for i in range(chunks):
|
||||
size = base + (1 if i < rem else 0)
|
||||
end = start + size
|
||||
group = all_sources[start:end]
|
||||
start = end
|
||||
|
||||
if not group:
|
||||
continue
|
||||
|
||||
unity_cpp = unity_dir / f"unity{i + 1}.cpp"
|
||||
# Use synchronous path to simplify error handling here
|
||||
result = self._compiler._compile_unity_sync(
|
||||
compile_opts,
|
||||
group,
|
||||
unity_output_path=unity_cpp,
|
||||
)
|
||||
if not result.ok:
|
||||
print(f"[BUILD API] Unity chunk {i + 1} failed: {result.stderr}")
|
||||
return False
|
||||
unity_objects.append(unity_cpp.with_suffix(".o"))
|
||||
|
||||
if not unity_objects:
|
||||
print("[BUILD API] No unity objects produced")
|
||||
return False
|
||||
|
||||
# Archive unity objects into the final library
|
||||
archive_result = self._compiler.create_archive(
|
||||
unity_objects, self._library_file, LibarchiveOptions()
|
||||
).result()
|
||||
if not archive_result.ok:
|
||||
print(f"[BUILD API] Archive creation failed: {archive_result.stderr}")
|
||||
return False
|
||||
|
||||
print(
|
||||
f"[BUILD API] Library built successfully (UNITY): {self._library_file}"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[BUILD API] UNITY library build failed with exception: {e}")
|
||||
return False
|
||||
|
||||
def _create_static_library(
|
||||
self, object_files: List[Path], output_lib: Path
|
||||
) -> bool:
|
||||
"""
|
||||
Create static library from object files.
|
||||
|
||||
Args:
|
||||
object_files: List of object files to archive
|
||||
output_lib: Output library file path
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not object_files:
|
||||
return False
|
||||
|
||||
# Use archiver from build flags
|
||||
archiver_cmd = self.build_flags.tools.archiver
|
||||
archive_flags = self.build_flags.archive.flags
|
||||
|
||||
# Build archiver command
|
||||
cmd = (
|
||||
archiver_cmd
|
||||
+ [archive_flags]
|
||||
+ [str(output_lib)]
|
||||
+ [str(f) for f in object_files]
|
||||
)
|
||||
|
||||
print(f"[BUILD API] Creating static library: {' '.join(cmd)}")
|
||||
|
||||
# Execute archiver command using stdout-pumping wrapper
|
||||
try:
|
||||
_completed = subprocess_run(
|
||||
command=cmd,
|
||||
cwd=None,
|
||||
check=True,
|
||||
timeout=900,
|
||||
enable_stack_trace=False,
|
||||
)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"[BUILD API] Archiver failed: {e}")
|
||||
print(f"[BUILD API] Archiver stdout: {e.stdout}")
|
||||
print(f"[BUILD API] Archiver stderr: {e.stderr}")
|
||||
return False
|
||||
|
||||
def build_targets(self, target_files: Sequence[str | Path]) -> List[BuildResult]:
|
||||
"""
|
||||
Build multiple targets efficiently.
|
||||
|
||||
Args:
|
||||
target_files: List of source files to build
|
||||
|
||||
Returns:
|
||||
List[BuildResult]: Results for each target
|
||||
"""
|
||||
print(
|
||||
f"[BUILD API] Building {len(target_files)} targets for {self.build_type.value}"
|
||||
)
|
||||
|
||||
# Ensure shared artifacts are built first
|
||||
if not self._ensure_pch_built():
|
||||
return [
|
||||
BuildResult(
|
||||
target=BuildTarget(Path(f), Path(f).stem, self.targets_dir),
|
||||
success=False,
|
||||
message="PCH build failed",
|
||||
build_time=0.0,
|
||||
)
|
||||
for f in target_files
|
||||
]
|
||||
|
||||
if not self._ensure_library_built():
|
||||
return [
|
||||
BuildResult(
|
||||
target=BuildTarget(Path(f), Path(f).stem, self.targets_dir),
|
||||
success=False,
|
||||
message="Library build failed",
|
||||
build_time=0.0,
|
||||
)
|
||||
for f in target_files
|
||||
]
|
||||
|
||||
# Build individual targets
|
||||
results: List[BuildResult] = []
|
||||
for target_file in target_files:
|
||||
result = self._build_single_target(Path(target_file))
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
def _build_single_target(self, target_file: Path) -> BuildResult:
|
||||
"""
|
||||
Build a single target.
|
||||
|
||||
Args:
|
||||
target_file: Source file to build
|
||||
|
||||
Returns:
|
||||
BuildResult: Result of the build
|
||||
"""
|
||||
import time
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Create build target
|
||||
target_dir = self.targets_dir / target_file.stem
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target = BuildTarget(
|
||||
source_file=target_file, name=target_file.stem, output_dir=target_dir
|
||||
)
|
||||
|
||||
print(f"[BUILD API] Building target: {target.name}")
|
||||
|
||||
try:
|
||||
# Compile and link the target
|
||||
success = self._compile_and_link_target(target)
|
||||
|
||||
build_time = time.time() - start_time
|
||||
message = "Build successful" if success else "Build failed"
|
||||
|
||||
return BuildResult(
|
||||
target=target, success=success, message=message, build_time=build_time
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
build_time = time.time() - start_time
|
||||
return BuildResult(
|
||||
target=target,
|
||||
success=False,
|
||||
message=f"Build error: {e}",
|
||||
build_time=build_time,
|
||||
)
|
||||
|
||||
def _compile_and_link_target(self, target: BuildTarget) -> bool:
|
||||
"""
|
||||
Compile and link a single target.
|
||||
|
||||
Args:
|
||||
target: Target to build
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not self._compiler:
|
||||
return False
|
||||
|
||||
# Compile target source file
|
||||
obj_file = target.output_dir / f"{target.name}.o"
|
||||
if target.source_file.suffix == ".cpp":
|
||||
success = self._compiler.compile_cpp_file(target.source_file, obj_file)
|
||||
else:
|
||||
# For .ino files
|
||||
success = self._compiler.compile_ino_file(target.source_file, obj_file)
|
||||
|
||||
if not success:
|
||||
print(f"[BUILD API] Failed to compile {target.source_file}")
|
||||
return False
|
||||
|
||||
# Link with library to create executable
|
||||
exe_file = target.get_output_executable()
|
||||
success = self._link_executable([obj_file], exe_file)
|
||||
|
||||
if not success:
|
||||
print(f"[BUILD API] Failed to link {exe_file}")
|
||||
return False
|
||||
|
||||
print(f"[BUILD API] Successfully built {exe_file}")
|
||||
return True
|
||||
|
||||
def _link_executable(self, object_files: List[Path], output_exe: Path) -> bool:
|
||||
"""
|
||||
Link object files into an executable.
|
||||
|
||||
Args:
|
||||
object_files: Object files to link
|
||||
output_exe: Output executable path
|
||||
|
||||
Returns:
|
||||
bool: True if successful, False otherwise
|
||||
"""
|
||||
if not self._library_file or not self._library_file.exists():
|
||||
print(f"[BUILD API] Library not available for linking")
|
||||
return False
|
||||
|
||||
# Build linker command
|
||||
linker_cmd = self.build_flags.tools.linker
|
||||
link_flags = self.build_flags.link_flags
|
||||
|
||||
cmd = (
|
||||
linker_cmd
|
||||
+ link_flags
|
||||
+ [str(f) for f in object_files]
|
||||
+ [str(self._library_file)]
|
||||
+ ["-o", str(output_exe)]
|
||||
)
|
||||
cmd_str = subprocess.list2cmdline(cmd)
|
||||
|
||||
print(f"[BUILD API] Linking: {cmd_str}")
|
||||
|
||||
# Execute linker using stdout-pumping wrapper
|
||||
try:
|
||||
_completed = subprocess_run(
|
||||
command=cmd,
|
||||
cwd=None,
|
||||
check=True,
|
||||
timeout=900,
|
||||
enable_stack_trace=False,
|
||||
)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"[BUILD API] Linker failed: {e}")
|
||||
print(f"[BUILD API] Linker stdout: {e.stdout}")
|
||||
print(f"[BUILD API] Linker stderr: {e.stderr}")
|
||||
return False
|
||||
|
||||
def get_build_info(self) -> Dict[str, Any]:
|
||||
"""Get information about the current build configuration"""
|
||||
return {
|
||||
"build_flags_toml": str(self.build_flags_toml),
|
||||
"build_dir": str(self.build_dir),
|
||||
"build_type": self.build_type.value,
|
||||
"use_pch": self.use_pch,
|
||||
"parallel": self.parallel,
|
||||
"pch_file": str(self._pch_file) if self._pch_file else None,
|
||||
"library_file": str(self._library_file) if self._library_file else None,
|
||||
}
|
||||
|
||||
def clean_build_artifacts(self) -> None:
|
||||
"""Clean all build artifacts"""
|
||||
print(f"[BUILD API] Cleaning build artifacts in {self.build_dir}")
|
||||
|
||||
if self.build_dir.exists():
|
||||
shutil.rmtree(self.build_dir)
|
||||
self.build_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Recreate subdirectories
|
||||
for directory in [self.artifacts_dir, self.targets_dir, self.cache_dir]:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Reset internal state
|
||||
self._pch_file = None
|
||||
self._library_file = None
|
||||
if self._fingerprint_cache:
|
||||
# Clear the cache by resetting it
|
||||
cache_file = self.cache_dir / "fingerprint_cache.json"
|
||||
self._fingerprint_cache = FingerprintCache(cache_file)
|
||||
|
||||
|
||||
def create_unit_test_builder(
|
||||
build_dir: str | Path = ".build/unit",
|
||||
use_pch: bool = True,
|
||||
parallel: bool = True,
|
||||
clean: bool = False,
|
||||
) -> BuildAPI:
|
||||
"""
|
||||
Create a BuildAPI instance for unit tests.
|
||||
|
||||
Args:
|
||||
build_dir: Build directory for unit test outputs
|
||||
use_pch: Whether to use precompiled headers
|
||||
parallel: Whether to enable parallel compilation
|
||||
clean: Whether to force a clean build
|
||||
|
||||
Returns:
|
||||
BuildAPI: Configured build API for unit tests
|
||||
"""
|
||||
build_flags_toml = PROJECT_ROOT / "ci" / "build_unit.toml"
|
||||
return BuildAPI(
|
||||
build_flags_toml=build_flags_toml,
|
||||
build_dir=build_dir,
|
||||
build_type=BuildType.UNIT_TEST,
|
||||
use_pch=use_pch,
|
||||
parallel=parallel,
|
||||
clean=clean,
|
||||
)
|
||||
|
||||
|
||||
def create_example_builder(
|
||||
build_dir: str | Path = ".build/examples",
|
||||
use_pch: bool = True,
|
||||
parallel: bool = True,
|
||||
clean: bool = False,
|
||||
) -> BuildAPI:
|
||||
"""
|
||||
Create a BuildAPI instance for examples.
|
||||
|
||||
Args:
|
||||
build_dir: Build directory for example outputs
|
||||
use_pch: Whether to use precompiled headers
|
||||
parallel: Whether to enable parallel compilation
|
||||
clean: Whether to force a clean build
|
||||
|
||||
Returns:
|
||||
BuildAPI: Configured build API for examples
|
||||
"""
|
||||
build_flags_toml = PROJECT_ROOT / "ci" / "build_example.toml"
|
||||
return BuildAPI(
|
||||
build_flags_toml=build_flags_toml,
|
||||
build_dir=build_dir,
|
||||
build_type=BuildType.EXAMPLE,
|
||||
use_pch=use_pch,
|
||||
parallel=parallel,
|
||||
clean=clean,
|
||||
)
|
||||
Reference in New Issue
Block a user