#!/usr/bin/env python3 """ FastLED Unified Build API This module provides a unified build system for both unit tests and examples, centralizing PCH and library building while providing a simple, elegant API. Key Features: - Unified API for both unit tests and examples - Automatic PCH header building and reuse - Automatic libfastled.a building and reuse - Organized build directories under .build/ - Efficient multiple target building - Simple, elegant interface Usage: # Unit test building unit_builder = BuildAPI( config_file="ci/build_unit.toml", build_dir=".build/unit", build_type=BuildType.UNIT_TEST ) unit_builder.build_targets(["test_json.cpp", "test_color.cpp"]) # Example building example_builder = BuildAPI( config_file="ci/build_example.toml", build_dir=".build/examples", build_type=BuildType.EXAMPLE ) example_builder.build_targets(["Blink.ino", "DemoReel100.ino"]) """ import os import shutil import subprocess import tempfile from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Set, Tuple from ci.ci.fingerprint_cache import FingerprintCache from ci.compiler.clang_compiler import ( BuildFlags, Compiler, CompilerOptions, LibarchiveOptions, ) from ci.compiler.test_example_compilation import get_fastled_core_sources from ci.util.paths import PROJECT_ROOT from ci.util.running_process import subprocess_run class BuildType(Enum): """Build type enumeration""" UNIT_TEST = "unit_test" EXAMPLE = "example" @dataclass class BuildTarget: """Represents a single build target (test file or example)""" source_file: Path name: str output_dir: Path def get_output_executable(self) -> Path: """Get the expected output executable path""" if self.source_file.suffix == ".cpp": # Unit test: test_name -> test_name.exe base_name = self.source_file.stem return self.output_dir / f"{base_name}.exe" else: # Example: Blink.ino -> Blink.exe base_name = self.source_file.stem return self.output_dir / f"{base_name}.exe" @dataclass class BuildResult: """Result of a build operation""" target: BuildTarget success: bool message: str build_time: float class BuildAPI: """ Unified build API for both unit tests and examples. This class centralizes all build operations including PCH building, library creation, and target compilation. It provides a simple, elegant interface while handling the complexity internally. """ def __init__( self, build_flags_toml: str | Path, build_dir: str | Path, build_type: BuildType, use_pch: bool = True, parallel: bool = True, clean: bool = False, ): """ Initialize the build API. Args: build_flags_toml: Path to build_flags.toml file (build_unit.toml or build_example.toml) build_dir: Directory for build outputs (e.g., .build/unit or .build/examples) build_type: Type of build (UNIT_TEST or EXAMPLE) use_pch: Whether to use precompiled headers parallel: Whether to enable parallel compilation clean: Whether to force a clean build """ self.build_flags_toml = Path(build_flags_toml) self.build_dir = Path(build_dir) self.build_type = build_type self.use_pch = use_pch self.parallel = parallel self.clean = clean # Ensure build directory exists self.build_dir.mkdir(parents=True, exist_ok=True) # Set up subdirectories for organization self.artifacts_dir = self.build_dir / "artifacts" # PCH, lib files self.targets_dir = self.build_dir / "targets" # Individual target builds self.cache_dir = self.build_dir / "cache" # Fingerprint caches for directory in [self.artifacts_dir, self.targets_dir, self.cache_dir]: directory.mkdir(parents=True, exist_ok=True) # Initialize internal state self._compiler: Optional[Compiler] = None self._pch_file: Optional[Path] = None self._library_file: Optional[Path] = None self._fingerprint_cache: Optional[FingerprintCache] = None # Parse build flags from TOML file self._parse_build_flags() # Create compiler from TOML configuration self._create_compiler() def _parse_build_flags(self) -> None: """Parse build flags from TOML file""" if not self.build_flags_toml.exists(): raise RuntimeError( f"Build flags TOML file not found: {self.build_flags_toml}" ) print(f"[BUILD API] Parsing build flags from {self.build_flags_toml}") # Parse build flags from TOML self.build_flags = BuildFlags.from_toml_file( self.build_flags_toml, quick_build=True, # Use quick build mode for faster iteration strict_mode=False, ) # Initialize fingerprint cache cache_file = self.cache_dir / "fingerprint_cache.json" self._fingerprint_cache = FingerprintCache(cache_file) def _create_compiler(self) -> None: """Create compiler from TOML configuration ONLY - no inline settings allowed""" from ci.compiler.clang_compiler import create_compiler_options_from_toml # ALL settings must come from the TOML file - NO hardcoded/inline settings allowed print(f"[BUILD API] Creating compiler from TOML configuration") # Set up PCH output path pch_output_path: Optional[str] = None if self.use_pch: pch_output_path = str( self.artifacts_dir / f"{self.build_type.value}_pch.hpp.pch" ) # Use the proper function to create CompilerOptions from TOML - NO inline settings self.compiler_settings = create_compiler_options_from_toml( toml_path=self.build_flags_toml, include_path=str(PROJECT_ROOT / "src"), # Base include path only quick_build=True, # Use quick build mode for faster iteration strict_mode=False, # Pass PCH and parallel settings from constructor use_pch=self.use_pch, pch_output_path=pch_output_path, parallel=self.parallel, ) # Create compiler instance with TOML-loaded settings self._compiler = Compiler(self.compiler_settings, self.build_flags) def _ensure_pch_built(self) -> bool: """ Ensure PCH header is built and ready. Returns: bool: True if PCH is ready, False otherwise """ if not self.use_pch or not self._compiler: return True print(f"[BUILD API] Building PCH header for {self.build_type.value}") # Check if we need to rebuild PCH if not self.clean and self._pch_file and self._pch_file.exists(): # TODO: Add proper dependency checking here print(f"[BUILD API] Reusing existing PCH: {self._pch_file}") return True # Build PCH success = self._compiler.create_pch_file() if success and self.compiler_settings.pch_output_path: self._pch_file = Path(self.compiler_settings.pch_output_path) print(f"[BUILD API] PCH built successfully: {self._pch_file}") else: print(f"[BUILD API] PCH build failed") return success def _ensure_library_built(self) -> bool: """ Ensure libfastled.a is built and ready. Returns: bool: True if library is ready, False otherwise """ if not self._compiler: return False lib_name = f"libfastled_{self.build_type.value}.a" self._library_file = self.artifacts_dir / lib_name print(f"[BUILD API] Building FastLED library for {self.build_type.value}") # Check if we need to rebuild library if not self.clean and self._library_file.exists(): # TODO: Add proper dependency checking here print(f"[BUILD API] Reusing existing library: {self._library_file}") return True # Prefer UNITY chunk build for faster library creation try: unity_dir = self.artifacts_dir / "unity" unity_dir.mkdir(parents=True, exist_ok=True) # Collect FastLED core sources (consistent with example compilation) all_sources = [ p for p in get_fastled_core_sources() if p.name != "stub_main.cpp" ] if not all_sources: print("[BUILD API] No FastLED sources found for unity build") return False # Deterministic sort relative to project root project_root = PROJECT_ROOT def sort_key(p: Path) -> str: try: rel = p.relative_to(project_root) except Exception: rel = p return rel.as_posix() all_sources = sorted(all_sources, key=sort_key) # Partition into chunks (1..4 typical) total = len(all_sources) max_chunks = min(4, max(1, (os.cpu_count() or 2) // 2)) chunks = max_chunks if total >= max_chunks else max(1, total) # Prepare compile options (PCH off for unity) compile_opts = CompilerOptions( include_path=self._compiler.settings.include_path, compiler=self._compiler.settings.compiler, defines=self._compiler.settings.defines, std_version=self._compiler.settings.std_version, compiler_args=self._compiler.settings.compiler_args, use_pch=False, additional_flags=["-c"], parallel=self.parallel, ) # Compute chunk sizes base = total // chunks rem = total % chunks start = 0 unity_objects: list[Path] = [] for i in range(chunks): size = base + (1 if i < rem else 0) end = start + size group = all_sources[start:end] start = end if not group: continue unity_cpp = unity_dir / f"unity{i + 1}.cpp" # Use synchronous path to simplify error handling here result = self._compiler._compile_unity_sync( compile_opts, group, unity_output_path=unity_cpp, ) if not result.ok: print(f"[BUILD API] Unity chunk {i + 1} failed: {result.stderr}") return False unity_objects.append(unity_cpp.with_suffix(".o")) if not unity_objects: print("[BUILD API] No unity objects produced") return False # Archive unity objects into the final library archive_result = self._compiler.create_archive( unity_objects, self._library_file, LibarchiveOptions() ).result() if not archive_result.ok: print(f"[BUILD API] Archive creation failed: {archive_result.stderr}") return False print( f"[BUILD API] Library built successfully (UNITY): {self._library_file}" ) return True except Exception as e: print(f"[BUILD API] UNITY library build failed with exception: {e}") return False def _create_static_library( self, object_files: List[Path], output_lib: Path ) -> bool: """ Create static library from object files. Args: object_files: List of object files to archive output_lib: Output library file path Returns: bool: True if successful, False otherwise """ if not object_files: return False # Use archiver from build flags archiver_cmd = self.build_flags.tools.archiver archive_flags = self.build_flags.archive.flags # Build archiver command cmd = ( archiver_cmd + [archive_flags] + [str(output_lib)] + [str(f) for f in object_files] ) print(f"[BUILD API] Creating static library: {' '.join(cmd)}") # Execute archiver command using stdout-pumping wrapper try: _completed = subprocess_run( command=cmd, cwd=None, check=True, timeout=900, enable_stack_trace=False, ) return True except subprocess.CalledProcessError as e: print(f"[BUILD API] Archiver failed: {e}") print(f"[BUILD API] Archiver stdout: {e.stdout}") print(f"[BUILD API] Archiver stderr: {e.stderr}") return False def build_targets(self, target_files: Sequence[str | Path]) -> List[BuildResult]: """ Build multiple targets efficiently. Args: target_files: List of source files to build Returns: List[BuildResult]: Results for each target """ print( f"[BUILD API] Building {len(target_files)} targets for {self.build_type.value}" ) # Ensure shared artifacts are built first if not self._ensure_pch_built(): return [ BuildResult( target=BuildTarget(Path(f), Path(f).stem, self.targets_dir), success=False, message="PCH build failed", build_time=0.0, ) for f in target_files ] if not self._ensure_library_built(): return [ BuildResult( target=BuildTarget(Path(f), Path(f).stem, self.targets_dir), success=False, message="Library build failed", build_time=0.0, ) for f in target_files ] # Build individual targets results: List[BuildResult] = [] for target_file in target_files: result = self._build_single_target(Path(target_file)) results.append(result) return results def _build_single_target(self, target_file: Path) -> BuildResult: """ Build a single target. Args: target_file: Source file to build Returns: BuildResult: Result of the build """ import time start_time = time.time() # Create build target target_dir = self.targets_dir / target_file.stem target_dir.mkdir(parents=True, exist_ok=True) target = BuildTarget( source_file=target_file, name=target_file.stem, output_dir=target_dir ) print(f"[BUILD API] Building target: {target.name}") try: # Compile and link the target success = self._compile_and_link_target(target) build_time = time.time() - start_time message = "Build successful" if success else "Build failed" return BuildResult( target=target, success=success, message=message, build_time=build_time ) except Exception as e: build_time = time.time() - start_time return BuildResult( target=target, success=False, message=f"Build error: {e}", build_time=build_time, ) def _compile_and_link_target(self, target: BuildTarget) -> bool: """ Compile and link a single target. Args: target: Target to build Returns: bool: True if successful, False otherwise """ if not self._compiler: return False # Compile target source file obj_file = target.output_dir / f"{target.name}.o" if target.source_file.suffix == ".cpp": success = self._compiler.compile_cpp_file(target.source_file, obj_file) else: # For .ino files success = self._compiler.compile_ino_file(target.source_file, obj_file) if not success: print(f"[BUILD API] Failed to compile {target.source_file}") return False # Link with library to create executable exe_file = target.get_output_executable() success = self._link_executable([obj_file], exe_file) if not success: print(f"[BUILD API] Failed to link {exe_file}") return False print(f"[BUILD API] Successfully built {exe_file}") return True def _link_executable(self, object_files: List[Path], output_exe: Path) -> bool: """ Link object files into an executable. Args: object_files: Object files to link output_exe: Output executable path Returns: bool: True if successful, False otherwise """ if not self._library_file or not self._library_file.exists(): print(f"[BUILD API] Library not available for linking") return False # Build linker command linker_cmd = self.build_flags.tools.linker link_flags = self.build_flags.link_flags cmd = ( linker_cmd + link_flags + [str(f) for f in object_files] + [str(self._library_file)] + ["-o", str(output_exe)] ) cmd_str = subprocess.list2cmdline(cmd) print(f"[BUILD API] Linking: {cmd_str}") # Execute linker using stdout-pumping wrapper try: _completed = subprocess_run( command=cmd, cwd=None, check=True, timeout=900, enable_stack_trace=False, ) return True except subprocess.CalledProcessError as e: print(f"[BUILD API] Linker failed: {e}") print(f"[BUILD API] Linker stdout: {e.stdout}") print(f"[BUILD API] Linker stderr: {e.stderr}") return False def get_build_info(self) -> Dict[str, Any]: """Get information about the current build configuration""" return { "build_flags_toml": str(self.build_flags_toml), "build_dir": str(self.build_dir), "build_type": self.build_type.value, "use_pch": self.use_pch, "parallel": self.parallel, "pch_file": str(self._pch_file) if self._pch_file else None, "library_file": str(self._library_file) if self._library_file else None, } def clean_build_artifacts(self) -> None: """Clean all build artifacts""" print(f"[BUILD API] Cleaning build artifacts in {self.build_dir}") if self.build_dir.exists(): shutil.rmtree(self.build_dir) self.build_dir.mkdir(parents=True, exist_ok=True) # Recreate subdirectories for directory in [self.artifacts_dir, self.targets_dir, self.cache_dir]: directory.mkdir(parents=True, exist_ok=True) # Reset internal state self._pch_file = None self._library_file = None if self._fingerprint_cache: # Clear the cache by resetting it cache_file = self.cache_dir / "fingerprint_cache.json" self._fingerprint_cache = FingerprintCache(cache_file) def create_unit_test_builder( build_dir: str | Path = ".build/unit", use_pch: bool = True, parallel: bool = True, clean: bool = False, ) -> BuildAPI: """ Create a BuildAPI instance for unit tests. Args: build_dir: Build directory for unit test outputs use_pch: Whether to use precompiled headers parallel: Whether to enable parallel compilation clean: Whether to force a clean build Returns: BuildAPI: Configured build API for unit tests """ build_flags_toml = PROJECT_ROOT / "ci" / "build_unit.toml" return BuildAPI( build_flags_toml=build_flags_toml, build_dir=build_dir, build_type=BuildType.UNIT_TEST, use_pch=use_pch, parallel=parallel, clean=clean, ) def create_example_builder( build_dir: str | Path = ".build/examples", use_pch: bool = True, parallel: bool = True, clean: bool = False, ) -> BuildAPI: """ Create a BuildAPI instance for examples. Args: build_dir: Build directory for example outputs use_pch: Whether to use precompiled headers parallel: Whether to enable parallel compilation clean: Whether to force a clean build Returns: BuildAPI: Configured build API for examples """ build_flags_toml = PROJECT_ROOT / "ci" / "build_example.toml" return BuildAPI( build_flags_toml=build_flags_toml, build_dir=build_dir, build_type=BuildType.EXAMPLE, use_pch=use_pch, parallel=parallel, clean=clean, )