From 89f5d006c9a206c00a26d77b0587e8b9427f00eb Mon Sep 17 00:00:00 2001 From: Raphael Coeffic Date: Sun, 27 Jul 2025 03:50:22 +0200 Subject: [PATCH] chore: use Ninja for Win cpn and improve Python support (#6455) --- .github/workflows/win_cpn-64.yml | 21 +- CMakeLists.txt | 38 ++-- cmake/GenericDefinitions.cmake | 12 +- companion/src/CMakeLists.txt | 9 +- radio/src/bootloader/CMakeLists.txt | 2 +- tools/build-companion.sh | 333 ++++++++++++++++++++++------ tools/build-gh.sh | 23 +- 7 files changed, 312 insertions(+), 126 deletions(-) diff --git a/.github/workflows/win_cpn-64.yml b/.github/workflows/win_cpn-64.yml index e5d562a54..212f0d068 100644 --- a/.github/workflows/win_cpn-64.yml +++ b/.github/workflows/win_cpn-64.yml @@ -50,24 +50,13 @@ jobs: - name: Install Dependencies run: | pacman -S --noconfirm mingw-w64-x86_64-cmake \ - mingw-w64-x86_64-python-pip \ - mingw-w64-x86_64-python-pillow \ - mingw-w64-x86_64-python-lz4 \ - mingw-w64-x86_64-libjpeg-turbo \ + mingw-w64-x86_64-ninja \ mingw-w64-x86_64-zlib \ - mingw-w64-x86_64-libtiff \ - mingw-w64-x86_64-freetype \ - mingw-w64-x86_64-lcms2 \ - mingw-w64-x86_64-libwebp \ - mingw-w64-x86_64-openjpeg2 \ - mingw-w64-x86_64-libimagequant \ - mingw-w64-x86_64-libraqm \ mingw-w64-x86_64-SDL2 \ mingw-w64-x86_64-clang \ mingw-w64-x86_64-nsis \ mingw-w64-x86_64-dfu-util \ mingw-w64-x86_64-openssl - python -m pip install clang jinja2 - name: Check out the repo uses: actions/checkout@v4 @@ -85,9 +74,15 @@ jobs: arch: ${{ env.MINGW_VERSION }} modules: 'qtmultimedia qtserialport' + - name: Install Python Dependencies + run: | + echo $PATH + $Python3_ROOT_DIR/python3.exe -m pip install clang jinja2 lz4 pillow + - name: Build working-directory: ${{github.workspace}} - # Execute the build. You can specify a specific target with "--target " + env: + CMAKE_GENERATOR: Ninja run: | mkdir output && \ CMAKE_PREFIX_PATH=$RUNNER_WORKSPACE/Qt/$QT_VERSION/$MINGW_PATH \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ad793010..44a5a169a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,55 +73,55 @@ if(EdgeTX_SUPERBUILD) -DNATIVE_BUILD:BOOL=0 INSTALL_COMMAND "" EXCLUDE_FROM_ALL TRUE - ) + ) add_custom_target(configure DEPENDS native-configure arm-none-eabi-configure) add_custom_target(libsimulator - COMMAND $(MAKE) -C native libsimulator + COMMAND ${CMAKE_COMMAND} --build native --target libsimulator --parallel DEPENDS native-configure - ) + ) add_custom_target(simulator - COMMAND $(MAKE) -C native simulator + COMMAND ${CMAKE_COMMAND} --build native --target simulator --parallel DEPENDS native-configure - ) + ) add_custom_target(simu - COMMAND $(MAKE) -C native simu + COMMAND ${CMAKE_COMMAND} --build native --target simu --parallel DEPENDS native-configure - ) + ) add_custom_target(companion - COMMAND $(MAKE) -C native companion + COMMAND ${CMAKE_COMMAND} --build native --target companion --parallel DEPENDS native-configure - ) + ) add_custom_target(gtests-radio - COMMAND $(MAKE) -C native gtests-radio + COMMAND ${CMAKE_COMMAND} --build native --target gtests-radio --parallel DEPENDS native-configure - ) + ) add_custom_target(tests-radio - COMMAND $(MAKE) -C native tests-radio + COMMAND ${CMAKE_COMMAND} --build native --target tests-radio --parallel DEPENDS native-configure - ) + ) add_custom_target(bootloader - COMMAND $(MAKE) -C arm-none-eabi bootloader + COMMAND ${CMAKE_COMMAND} --build arm-none-eabi --target bootloader --parallel DEPENDS arm-none-eabi-configure - ) + ) add_custom_target(firmware - COMMAND $(MAKE) -C arm-none-eabi firmware + COMMAND ${CMAKE_COMMAND} --build arm-none-eabi --target firmware --parallel DEPENDS arm-none-eabi-configure - ) + ) add_custom_target(firmware-size - COMMAND $(MAKE) -C arm-none-eabi firmware-size + COMMAND ${CMAKE_COMMAND} --build arm-none-eabi --target firmware-size --parallel DEPENDS arm-none-eabi-configure - ) + ) # Quit cmake at this point return() diff --git a/cmake/GenericDefinitions.cmake b/cmake/GenericDefinitions.cmake index b7737be56..99850cc64 100644 --- a/cmake/GenericDefinitions.cmake +++ b/cmake/GenericDefinitions.cmake @@ -50,10 +50,14 @@ include(Macros) git_id(GIT_STR) # Python check -find_package(PythonInterp 3 REQUIRED) -if(PYTHONINTERP_FOUND) - message(STATUS "Python found, version: ${PYTHON_VERSION_STRING}") - get_filename_component(PYTHON_DIRECTORY ${PYTHON_EXECUTABLE} DIRECTORY) +set(Python3_FIND_VIRTUALENV FIRST) +set(Python3_FIND_STRATEGY LOCATION) + +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +if(Python3_Interpreter_FOUND) + message(STATUS "Python found, version: ${Python3_VERSION}") + cmake_path(NATIVE_PATH Python3_EXECUTABLE PYTHON_EXECUTABLE) else() message(WARNING "Python not found! Most firmware and simu flavors not buildable.") set(LUA NO) diff --git a/companion/src/CMakeLists.txt b/companion/src/CMakeLists.txt index 2fe7e7489..1ddfb34d0 100644 --- a/companion/src/CMakeLists.txt +++ b/companion/src/CMakeLists.txt @@ -141,10 +141,11 @@ set(HWDEFS_CMD "${COMPANION_SRC_DIRECTORY}/../util/generate_hwdefs_qrc.py") # qrc used by Companion and Simulator add_custom_command(OUTPUT ${HWDEFS_QRC} - COMMAND ${HWDEFS_CMD} -d ${HWDEFS_DIR} - -t ${HWDEFS_TMPL} - -p ${HWDEFS_PHDR} - -o ${HWDEFS_QRC} + COMMAND ${PYTHON_EXECUTABLE} ${HWDEFS_CMD} + -d ${HWDEFS_DIR} + -t ${HWDEFS_TMPL} + -p ${HWDEFS_PHDR} + -o ${HWDEFS_QRC} DEPENDS ${HWDEFS_TMPL} ) diff --git a/radio/src/bootloader/CMakeLists.txt b/radio/src/bootloader/CMakeLists.txt index 908e7bbc1..f83a98689 100644 --- a/radio/src/bootloader/CMakeLists.txt +++ b/radio/src/bootloader/CMakeLists.txt @@ -102,7 +102,7 @@ add_custom_command( COMMAND ${CMAKE_OBJCOPY} -O binary bootloader.elf ${CMAKE_BINARY_DIR}/bootloader.bin ) -if(PYTHONINTERP_FOUND) +if(Python3_FOUND) add_custom_command( TARGET bootloader POST_BUILD COMMAND ${PYTHON_EXECUTABLE} ${RADIO_DIRECTORY}/util/bin2lbm.py ${CMAKE_BINARY_DIR}/bootloader.bin bootloader.lbm diff --git a/tools/build-companion.sh b/tools/build-companion.sh index f0329ed3d..5779db28c 100755 --- a/tools/build-companion.sh +++ b/tools/build-companion.sh @@ -1,39 +1,28 @@ #!/bin/bash -# Stops on first error, echo on -set -e -set -x - SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/build-common.sh" -if [ "$(uname)" = "Darwin" ]; then - num_cpus=$(sysctl -n hw.ncpu) - : "${JOBS:=$num_cpus}" -else - JOBS=3 -fi - -while [ $# -gt 0 ] -do - case "$1" in - --jobs=*) - JOBS="${1#*=}";; - -j*) - JOBS="${1#*j}";; - -*) - echo >&2 "usage: $0 [-j|--jobs=] SRCDIR OUTDIR" - exit 1;; - *) - break;; # terminate while loop - esac - shift -done - SRCDIR=$1 OUTDIR=$2 -COMMON_OPTIONS="-DGVARS=YES -DHELI=YES -DLUA=YES -Wno-dev -DCMAKE_BUILD_TYPE=Release" +if [[ -z ${SRCDIR} ]]; then + SRCDIR="$(pwd)" +fi + +if [[ -z ${OUTDIR} ]]; then + OUTDIR="$(pwd)/output" +fi + +QUIET_FLAGS="" +if [[ "$CMAKE_GENERATOR" == "Ninja" ]]; then + QUIET_FLAGS="-- --quiet" +else + # Assume Makefile generator for non-Ninja builds + COMMON_OPTIONS="-DCMAKE_RULE_MESSAGES=OFF" +fi + +COMMON_OPTIONS="${COMMON_OPTIONS} -DCMAKE_BUILD_TYPE=Release -DCMAKE_MESSAGE_LOG_LEVEL=WARNING -Wno-dev -DGVARS=YES -DHELI=YES -DLUA=YES" if [ "$(uname)" = "Darwin" ]; then COMMON_OPTIONS="${COMMON_OPTIONS} -DCMAKE_OSX_DEPLOYMENT_TARGET='10.15'" elif [ "$(uname)" != "Linux" ]; then # Assume Windows and MSYS2 @@ -62,46 +51,262 @@ if [[ -z ${EDGETX_VERSION_SUFFIX} ]]; then fi fi -rm -rf build -mkdir build -cd build +if [ "$(uname)" = "Linux" ] && [ -n "$GITHUB_ACTIONS" ]; then + MAX_JOBS=3 +fi -declare -a simulator_plugins=(x9lite x9lites - x7 x7access - t8 t12 t12max tx12 tx12mk2 - zorro commando8 boxer pocket mt12 gx12 - tlite tpro tprov2 tpros bumblebee lr3pro t14 - x9d x9dp x9dp2019 x9e - xlite xlites - nv14 el18 pl18 pl18ev pl18u st16 - x10 x10express x12s - t15 t16 t18 t20 t20v2 tx16s f16 v14 v16) +rm -rf build && mkdir build && cd build -for plugin in "${simulator_plugins[@]}" -do - BUILD_OPTIONS="${COMMON_OPTIONS} " - - echo "Building ${plugin}" +# Function to output error logs (works in both GitHub Actions and terminal) +output_error_log() { + local log_file="$1" + local context="$2" - if ! get_target_build_options "$plugin"; then - echo "Error: Failed to find a match for target '$plugin'" - exit 1 + if [[ -f "$log_file" ]]; then + echo "------------------------------------------" + echo " Full error output from $log_file:" + echo "------------------------------------------" + cat "$log_file" + else + echo "⚠️ Warning: Log file $log_file not found for $context" + fi +} + +# Function to show last N lines of a log file for warnings +show_log_summary() { + local log_file="$1" + local lines="${2:-50}" + local context="$3" + + if [[ -f "$log_file" ]]; then + echo "------------------------------------------" + echo "Last $lines lines from $log_file:" + echo "------------------------------------------" + tail -n "$lines" "$log_file" + fi +} + +run_pipeline() { + local pipeline_type="$1" + local log_file="${2:-/dev/null}" + local context="$3" + local show_details="${4:-false}" + local cmake_opts="--parallel ${MAX_JOBS} ${QUIET_FLAGS}" + + case "$pipeline_type" in + "plugin") + clean_build + if ! execute_with_output "⚙️ Configuring" "cmake ${BUILD_OPTIONS} ${SRCDIR}" "$log_file" "$show_details"; then + output_error_log "$log_file" "$context (Configuration)" + return 1 + fi + if ! execute_with_output "🔧 Native config" "cmake --build . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then + output_error_log "$log_file" "$context (Native Configure)" + return 1 + fi + if ! execute_with_output "📦 Building lib" "cmake --build native --target libsimulator ${cmake_opts}" "$log_file" "$show_details"; then + output_error_log "$log_file" "$context (Library Build)" + return 1 + fi + ;; + "final") + if ! execute_with_output "🔧 Final config" "cmake --build . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then + output_error_log "$log_file" "Final Configuration" + return 1 + fi + if ! execute_with_output "📦 Building companion" "cmake --build native --target companion ${cmake_opts}" "$log_file" "$show_details"; then + output_error_log "$log_file" "$context (Companion Build)" + return 1 + fi + if ! execute_with_output "📦 Packaging" "cmake --build native --target ${PACKAGE_TARGET}" "$log_file" "true"; then + output_error_log "$log_file" "Final Packaging" + return 1 + fi + ;; + esac + + return 0 +} + +execute_with_output() { + local description="$1" + local command="$2" + local log_file="$3" + local show_output="${4:-false}" + + if [[ "$show_output" == "true" && "$log_file" != "/dev/null" ]]; then + echo " $description..." + fi + + eval "$command" >> "$log_file" 2>&1 +} + +clean_build() { + rm -f CMakeCache.txt native/CMakeCache.txt +} + +get_platform_config() { + local platform=$(uname) + case "$platform" in + "Darwin") + PACKAGE_TARGET="package" + PACKAGE_FILES="*.dmg" + PACKAGE_NAME="macOS DMG" + PACKAGE_EMOJI="🍎" + ;; + "Linux") + PACKAGE_TARGET="package" + PACKAGE_FILES="*.AppImage" + PACKAGE_NAME="Linux AppImage" + PACKAGE_EMOJI="🐧" + ;; + *) + PACKAGE_TARGET="installer" + PACKAGE_FILES="companion/*.exe" + PACKAGE_NAME="Windows installer" + PACKAGE_EMOJI="🪟" + ;; + esac +} + +# Enhanced plugin builder with better error handling +build_plugin() { + local plugin="$1" + local log_file="build_${plugin}.log" + local verbose="${2:-false}" + + BUILD_OPTIONS="${COMMON_OPTIONS} " + + if ! get_target_build_options "$plugin" >> "$log_file" 2>&1; then + if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::error::Failed to get build options for $plugin" + fi + output_error_log "$log_file" "$plugin (Build Options)" + return 1 + fi + + # Only show detailed output in GitHub Actions or if verbose is requested + local show_details="false" + if [[ -n "$GITHUB_ACTIONS" || "$verbose" == "true" ]]; then + show_details="true" + fi + + if ! run_pipeline "plugin" "$log_file" "$plugin" "$show_details"; then + return 1 + fi + + # Check for warnings and show summary if found + if grep -q -i "warning" "$log_file"; then + echo " ⚠️ $plugin completed with warnings" + if [[ "$show_details" == "true" ]]; then + show_log_summary "$log_file" 50 "$plugin (Warnings)" + fi + return 0 fi - rm -f CMakeCache.txt native/CMakeCache.txt - cmake ${BUILD_OPTIONS} "${SRCDIR}" - cmake --build . --target native-configure - cmake --build native -j"${JOBS}" --target libsimulator -done + echo " ✅ $plugin completed successfully" + return 0 +} -cmake --build . --target native-configure -if [ "$(uname)" = "Darwin" ]; then - cmake --build native -j"${JOBS}" --target package - cp native/*.dmg "${OUTDIR}" -elif [ "$(uname)" = "Linux" ]; then - cmake --build native -j"${JOBS}" --target package - cp native/*.AppImage "${OUTDIR}" -else - cmake --build native --target installer - cp native/companion/*.exe "${OUTDIR}" +declare -a simulator_plugins=( + x9lite x9lites x9d x9dp x9dp2019 x9e + x7 x7access + t8 t12 t12max tx12 tx12mk2 t15 t16 t18 t20 t20v2 + xlite xlites + x10 x10express x12s + zorro tx16s + commando8 boxer pocket mt12 gx12 + tlite tpro tprov2 tpros bumblebee lr3pro t14 + nv14 el18 pl18 pl18ev pl18u st16 + f16 v14 v16 +) + +get_platform_config +TOTAL=${#simulator_plugins[@]} +FAILED_PLUGINS=() + +echo "🔨 Building $TOTAL Plugins for $PACKAGE_NAME" + +for i in "${!simulator_plugins[@]}"; do + plugin="${simulator_plugins[$i]}" + current=$((i + 1)) + percent=$((current * 100 / TOTAL)) + + # For terminal output, put each plugin on its own line for cleaner display + if [[ -n "$GITHUB_ACTIONS" ]]; then + printf "::group::%s %-12s [%2d/%2d] %3d%%\n" "$PACKAGE_EMOJI" "$plugin" "$current" "$TOTAL" "$percent" + else + echo "🔨 [$current/$TOTAL] ($percent%) Building $plugin..." + fi + + error_status=0 + if ! build_plugin "$plugin"; then + FAILED_PLUGINS+=("$plugin") + error_status=1 + fi + + if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::endgroup::" + fi + + if [ $error_status -ne 0 ]; then + # TODO: if not '--build-all' used + exit 1 + fi +done + +# Show summary of any failures (if using continue-on-error approach) +if [ ${#FAILED_PLUGINS[@]} -gt 0 ]; then + if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::group::❌ Build Failures Summary" + else + echo "❌ Build Failures Summary" + echo "==========================================" + fi + + echo "The following plugins failed to build:" + for failed_plugin in "${FAILED_PLUGINS[@]}"; do + echo " ❌ $failed_plugin" + done + + if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::endgroup::" + else + echo "==========================================" + fi + exit 1 fi + +if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::group::📦 Final $PACKAGE_NAME Package" +else + echo "📦 Final $PACKAGE_NAME Package" + echo "==========================================" +fi + +error_status=0 +if run_pipeline "final" "final.log" "companion" "true"; then + if cp native/$PACKAGE_FILES "${OUTDIR}" 2>/dev/null; then + echo " ✅ All builds completed successfully!" + echo " 📁 Package saved to: ${OUTDIR}" + else + echo " ❌ Failed to copy package files to output directory" + echo " 📁 Directory Contents:" + echo " ----------------------" + + echo "Contents of native/ directory:" + ls -la native/ || echo "native/ directory not found" + echo "Looking for files matching: $PACKAGE_FILES" + find native/ -name "$PACKAGE_FILES" 2>/dev/null || echo "No matching files found" + error_status=1 + fi +else + echo " ❌ Final packaging failed" + error_status=1 +fi + +if [[ -n "$GITHUB_ACTIONS" ]]; then + echo "::endgroup::" +fi + +exit $error_status diff --git a/tools/build-gh.sh b/tools/build-gh.sh index fc006054f..9fe504cd4 100755 --- a/tools/build-gh.sh +++ b/tools/build-gh.sh @@ -7,17 +7,6 @@ set -x SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/build-common.sh" -# Allow variable core usage -# default uses all cpu cores -# -if [ -f /usr/bin/nproc ]; then - num_cpus=$(nproc) -elif [ -f /usr/sbin/sysctl ]; then - num_cpus=$(sysctl -n hw.logicalcpu) -else - num_cpus=2 -fi -: "${CORES:=$num_cpus}" # If no build target, exit #: "${FLAVOR:=ALL}" @@ -25,14 +14,6 @@ fi for i in "$@" do case $i in - --jobs=*) - CORES="${i#*=}" - shift - ;; - -j*) - CORES="${i#*j}" - shift - ;; -Wno-error) WERROR=0 shift @@ -100,8 +81,8 @@ do fi cmake ${BUILD_OPTIONS} "${SRCDIR}" - cmake --build . --target arm-none-eabi-configure - cmake --build arm-none-eabi -j"${CORES}" --target ${FIRMARE_TARGET} + cmake --build . --target arm-none-eabi-configure --parallel + cmake --build arm-none-eabi --target ${FIRMARE_TARGET} --parallel rm -f CMakeCache.txt arm-none-eabi/CMakeCache.txt