diff --git a/.github/workflows/build_fw.yml b/.github/workflows/build_fw.yml index 2523ccefb..f816c69a8 100644 --- a/.github/workflows/build_fw.yml +++ b/.github/workflows/build_fw.yml @@ -73,11 +73,11 @@ jobs: - name: Test ${{ matrix.target }} env: FLAVOR: ${{ matrix.target }} + CMAKE_BUILD_PARALLEL_LEVEL: 4 run: | echo "Running commit tests" ./tools/commit-tests.sh - build: name: Run builds needs: test @@ -123,6 +123,7 @@ jobs: - name: Build ${{ matrix.target }} env: FLAVOR: ${{ matrix.target }} + CMAKE_BUILD_PARALLEL_LEVEL: 4 run: | echo "Running firmware build" ./tools/build-gh.sh diff --git a/.github/workflows/linux_cpn.yml b/.github/workflows/linux_cpn.yml index 9f2aed97f..7b94df47a 100644 --- a/.github/workflows/linux_cpn.yml +++ b/.github/workflows/linux_cpn.yml @@ -45,6 +45,8 @@ jobs: - name: Build working-directory: ${{github.workspace}} shell: bash + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 run: | mkdir output && mkdir logs && \ tools/build-companion.sh "$(pwd)" "$(pwd)/output/" 2>&1 | tee logs/build-main.log diff --git a/.github/workflows/macosx_cpn.yml b/.github/workflows/macosx_cpn.yml index 28fcfd61d..c553fe766 100644 --- a/.github/workflows/macosx_cpn.yml +++ b/.github/workflows/macosx_cpn.yml @@ -73,6 +73,8 @@ jobs: - name: Build working-directory: ${{github.workspace}} shell: bash + env: + CMAKE_BUILD_PARALLEL_LEVEL: 3 run: | mkdir output && mkdir logs && \ tools/build-companion.sh "$(pwd)" "$(pwd)/output/" 2>&1 | tee logs/build-main.log diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2c838b27f..060054150 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -51,6 +51,7 @@ jobs: env: FLAVOR: ${{ matrix.target }} EDGETX_VERSION_SUFFIX: nightly + CMAKE_BUILD_PARALLEL_LEVEL: 4 run: ./tools/build-gh.sh - name: Package firmware ${{ matrix.target }} diff --git a/.github/workflows/win_cpn-64.yml b/.github/workflows/win_cpn-64.yml index d571aa8b0..cce40805a 100644 --- a/.github/workflows/win_cpn-64.yml +++ b/.github/workflows/win_cpn-64.yml @@ -74,6 +74,8 @@ jobs: uses: ilammy/msvc-dev-cmd@v1 - name: Build + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 run: | mkdir output && mkdir logs && \ CMAKE_PREFIX_PATH="$QT_ROOT_DIR;$SDL2_ROOT" \ diff --git a/tools/build-common.sh b/tools/build-common.sh index 25f124ff6..69f07e57b 100644 --- a/tools/build-common.sh +++ b/tools/build-common.sh @@ -167,3 +167,44 @@ get_target_build_options() { ;; esac } + +# Determine parallel job limit based on environment +determine_max_jobs() { + if [[ -n ${CMAKE_BUILD_PARALLEL_LEVEL} ]]; then + MAX_JOBS=${CMAKE_BUILD_PARALLEL_LEVEL} + elif [ -n "$GITHUB_ACTIONS" ]; then + # Limit jobs in GitHub Actions to n-1 to avoid resource contention + if [ "$(uname)" = "Darwin" ]; then + MAX_JOBS=2 # macOS runners have 3 cores + else + MAX_JOBS=3 # Linux and Windows runners have 4 cores + fi + else + MAX_JOBS="" # Let CMake/build system decide + fi + export MAX_JOBS +} + +# Helper function to run cmake build with appropriate parallelism +cmake_build_parallel() { + local args=() + local native_flags="" + + # Separate cmake args from native flags (anything after --) + for arg in "$@"; do + if [[ "$arg" == "--" ]]; then + # Capture everything after -- as native flags + shift + native_flags="-- $*" + break + fi + args+=("$arg") + shift + done + + if [[ -n ${MAX_JOBS} ]]; then + cmake --build "${args[@]}" --parallel ${MAX_JOBS} ${native_flags} + else + cmake --build "${args[@]}" --parallel ${native_flags} + fi +} diff --git a/tools/build-companion.sh b/tools/build-companion.sh index 2a082dbbe..230249a5e 100755 --- a/tools/build-companion.sh +++ b/tools/build-companion.sh @@ -14,6 +14,9 @@ if [[ -z ${OUTDIR} ]]; then OUTDIR="$(pwd)/output" fi +# Determine parallel jobs +determine_max_jobs + # Create a master log file for the entire build process MASTER_LOG="build-summary.log" @@ -48,10 +51,6 @@ if [[ -z ${EDGETX_VERSION_SUFFIX} ]]; then fi fi -if [ "$(uname)" = "Linux" ] && [ -n "$GITHUB_ACTIONS" ]; then - MAX_JOBS=3 -fi - rm -rf build && mkdir build && cd build # Function to output error logs (works in both GitHub Actions and terminal) @@ -97,7 +96,7 @@ run_pipeline() { local log_file="${2:-/dev/null}" local context="$3" local show_details="${4:-false}" - local cmake_opts="--parallel ${MAX_JOBS} ${QUIET_FLAGS}" + local cmake_opts="${QUIET_FLAGS}" case "$pipeline_type" in "plugin") @@ -106,25 +105,25 @@ run_pipeline() { output_error_log "$log_file" "$context (Configuration)" return 1 fi - if ! execute_with_output "🔧 Native config" "cmake --build . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then + if ! execute_with_output "🔧 Native config" "cmake_build_parallel . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then output_error_log "$log_file" "$context (Native Configure)" return 1 fi - if ! execute_with_output "📦 Building lib" "cmake --build native --target libsimulator ${cmake_opts}" "$log_file" "$show_details"; then + if ! execute_with_output "📦 Building lib" "cmake_build_parallel native --target libsimulator ${cmake_opts}" "$log_file" "$show_details"; then output_error_log "$log_file" "$context (Library Build)" return 1 fi ;; "final") - if ! execute_with_output "🔧 Final config" "cmake --build . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then + if ! execute_with_output "🔧 Final config" "cmake_build_parallel . --target native-configure ${cmake_opts}" "$log_file" "$show_details"; then output_error_log "$log_file" "Final Configuration" return 1 fi - if ! execute_with_output "📦 Building companion" "cmake --build native --target companion ${cmake_opts}" "$log_file" "$show_details"; then + if ! execute_with_output "📦 Building companion" "cmake_build_parallel native --target companion ${cmake_opts}" "$log_file" "$show_details"; then output_error_log "$log_file" "$context (Companion Build)" return 1 fi - if ! execute_with_output "📦 Packaging" "cmake --build native --target ${PACKAGE_TARGET}" "$log_file" "true"; then + if ! execute_with_output "📦 Packaging" "cmake_build_parallel native --target ${PACKAGE_TARGET}" "$log_file" "true"; then output_error_log "$log_file" "Final Packaging" return 1 fi diff --git a/tools/build-gh.sh b/tools/build-gh.sh index ff1f58e42..262d6809c 100755 --- a/tools/build-gh.sh +++ b/tools/build-gh.sh @@ -1,16 +1,12 @@ #!/bin/bash -# Stops on first error, echo on +# Stop on first error, echo on set -e set -x SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/build-common.sh" - -# If no build target, exit -#: "${FLAVOR:=ALL}" - for i in "$@" do case $i in @@ -58,6 +54,9 @@ COMMON_OPTIONS+=${EXTRA_OPTIONS}" " : "${FIRMARE_TARGET:="firmware-size"}" +# Determine parallel jobs +determine_max_jobs + # workaround for GH repo owner git config --global --add safe.directory "$(pwd)" @@ -81,8 +80,9 @@ do fi cmake ${BUILD_OPTIONS} "${SRCDIR}" - cmake --build . --target arm-none-eabi-configure --parallel ${MAX_JOBS} - cmake --build arm-none-eabi --target ${FIRMARE_TARGET} --parallel ${MAX_JOBS} + + cmake_build_parallel . --target arm-none-eabi-configure + cmake_build_parallel arm-none-eabi --target ${FIRMARE_TARGET} rm -f CMakeCache.txt arm-none-eabi/CMakeCache.txt diff --git a/tools/commit-tests.sh b/tools/commit-tests.sh index c5ce23576..0d2e7802d 100755 --- a/tools/commit-tests.sh +++ b/tools/commit-tests.sh @@ -1,38 +1,15 @@ #!/bin/bash -# Stops on first error, echo on +# Stop on first error, echo on set -e set -x SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/build-common.sh" -# Allow variable core usage -# default uses all cpu cores -# -if [ -f /usr/bin/nproc ]; then - num_cpus=$(nproc) -elif [ -f /usr/sbin/sysctl ]; then - num_cpus=$(sysctl -n hw.logicalcpu) -else - num_cpus=2 -fi -: "${CORES:=$num_cpus}" - -# If no build target, exit -#: "${FLAVOR:=ALL}" - for i in "$@" do case $i in - --jobs=*) - CORES="${i#*=}" - shift - ;; - -j*) - CORES="${i#*j}" - shift - ;; -Wno-error) WERROR=0 shift @@ -57,10 +34,13 @@ if (( $WERROR )); then COMMON_OPTIONS+=" -DWARNINGS_AS_ERRORS=YES "; fi : ${EXTRA_OPTIONS:="$EXTRA_OPTIONS"} -COMMON_OPTIONS+=${EXTRA_OPTIONS} +COMMON_OPTIONS+=${EXTRA_OPTIONS}" " : ${FIRMARE_TARGET:="firmware-size"} +# Determine parallel jobs +determine_max_jobs + # wipe build directory clean rm -rf build && mkdir -p build && cd build @@ -79,12 +59,12 @@ do cmake ${BUILD_OPTIONS} "${SRCDIR}" - cmake --build . --target arm-none-eabi-configure - cmake --build arm-none-eabi -j"${CORES}" --target ${FIRMARE_TARGET} + cmake_build_parallel . --target arm-none-eabi-configure + cmake_build_parallel arm-none-eabi --target ${FIRMARE_TARGET} - cmake --build . --target native-configure - cmake --build native -j"${CORES}" --target libsimulator - cmake --build native -j"${CORES}" --target tests-radio + cmake_build_parallel . --target native-configure + cmake_build_parallel native --target libsimulator + cmake_build_parallel native --target tests-radio rm -f CMakeCache.txt native/CMakeCache.txt arm-none-eabi/CMakeCache.txt done