name: Build on: # There was no pull_request trigger here: a change landed unbuilt and the # nightly found out the next morning, or a user did. Building all 107 devices # per PR is what made that unaffordable, so .github/scripts/ci-matrix.py picks # only the devices a PR can reach. ready_for_review is not in the default set # (opened, synchronize, reopened), and the selector skips drafts, so without it # a PR opened as a draft would never build once marked ready. pull_request: branches: - master types: [opened, synchronize, reopened, ready_for_review] schedule: - cron: '0 3 * * *' workflow_dispatch: inputs: platform: description: Platform required: false # Cancel a PR's earlier run when a new commit lands on it. Scheduled and # dispatched runs are deliberately left alone: they write releases, and a # cancelled publish leaves the nightly half-uploaded. concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: # Which devices can this change actually reach? Only ever narrows a # pull_request -- the nightly is decided by firmware HEAD, not by this repo's # diff, so every other event gets the whole matrix. # # The narrowing happens HERE, in a job that always runs, and never in a # `paths:` filter on the trigger: a filtered-out workflow does not run at all, # so a required check would stay pending forever and the PR could never merge. select: name: Select devices # Same guard as preflight below. if: >- github.repository == 'OpenIPC/builder' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && !github.event.repository.private) runs-on: ubuntu-latest outputs: matrix: ${{ steps.pick.outputs.matrix }} needs-build: ${{ steps.pick.outputs.needs-build }} steps: - uses: actions/checkout@v4 - name: Check the selector still describes the tree run: python3 .github/scripts/ci-matrix.py --self-test - id: pick name: Pick the affected devices env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: python3 .github/scripts/ci-matrix.py preflight: name: Preflight # Guard rail for clones of this repo. A clone pushed to a new repository # (a private mirror, an internal re-host) inherits this file along with # its cron, and the full platform matrix then runs on that owner's Actions # bill, nightly and unattended, publishing nightlies nobody reads. Forks # are partially covered (GitHub disables scheduled workflows there), a # fresh repository is not. Mirrors OpenIPC/firmware#2255. # # Unattended runs happen only on the canonical repo. Anywhere else a # workflow runs only if someone explicitly asked for it, or if it is both # free and attended -- which is what the pull_request clause covers, now # that this workflow has such a trigger. if: >- github.repository == 'OpenIPC/builder' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && !github.event.repository.private) runs-on: ubuntu-latest outputs: should_build: ${{ steps.gate.outputs.should_build }} head_sha: ${{ steps.gate.outputs.head_sha }} short_sha: ${{ steps.gate.outputs.short_sha }} build_id: ${{ steps.gate.outputs.build_id }} built_at: ${{ steps.gate.outputs.built_at }} steps: - uses: actions/checkout@v4 - id: gate env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | HEAD=$(git rev-parse HEAD) SHORT=$(git rev-parse --short HEAD) BUILD_ID="nightly-$(date -u +%Y%m%d)-${SHORT}" BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) # Nightly rebuilds unconditionally on schedule. builder.sh re-clones # OpenIPC/firmware at HEAD on every run, so the relevant inputs are # mostly outside this repo (firmware HEAD, toolchain release, kernel # tarball, vendor osdrv, individual package upstreams). The previous # gate skipped when only this repo's HEAD matched the last published # nightly, which made upstream fixes (e.g. an openhisilicon mipi_rx # regression fix that lands in firmware) invisible to users until # something here happened to change. Always build on schedule. echo "Build: $BUILD_ID (event=${{ github.event_name }})" echo "should_build=true" >> "$GITHUB_OUTPUT" echo "head_sha=$HEAD" >> "$GITHUB_OUTPUT" echo "short_sha=$SHORT" >> "$GITHUB_OUTPUT" echo "build_id=$BUILD_ID" >> "$GITHUB_OUTPUT" echo "built_at=$BUILT_AT" >> "$GITHUB_OUTPUT" buildroot: name: Firmware needs: [preflight, select] if: >- needs.preflight.outputs.should_build == 'true' && needs.select.outputs.needs-build == 'true' runs-on: ubuntu-latest env: BUILD_ID: ${{ needs.preflight.outputs.build_id }} BUILD_SHA: ${{ needs.preflight.outputs.head_sha }} BUILD_PLATFORM: ${{ matrix.platform }} strategy: fail-fast: false # The device list lives in .github/scripts/ci-matrix.py, which is # also what narrows it. Everything except pull_request gets all of it. matrix: ${{ fromJson(needs.select.outputs.matrix) }} steps: - name: Checkout source uses: actions/checkout@v4 - name: Prepare firmware run: | echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf echo CACHE_DATE=$(date +%m) >> ${GITHUB_ENV} INPUT=${{inputs.platform}} MATRIX=$(echo ${{matrix.platform}} | cut -d_ -f1) if [ -z ${INPUT} ] || [ ${INPUT} = ${MATRIX} ]; then echo RUN=${MATRIX} >> ${GITHUB_ENV} fi - name: Setup ccache if: env.RUN uses: actions/cache@v4 with: path: /tmp/ccache key: ${{matrix.platform}}-${{env.CACHE_DATE}} # builder.sh does `rm -rf openipc` at the top of every run to force a # clean re-clone of OpenIPC/firmware, which also nukes the default # `openipc/output/dl/` directory Buildroot uses for downloaded source # tarballs. Without a cache, every cron run re-downloads ~1-3 GB of # tarballs from upstream mirrors — and any one of them randomly # failing (e.g. cgit `snapshot/` URLs that regenerate tarballs per # request with shifting sha256, like wireguard-tools on # git.zx2c4.com) takes down the matrix entry for that board. See # kaeru wireguard-zx2c4-tarball-regeneration-hash-drift for the # specific incident this caching addresses. # # Cache to /tmp/builder-dl (outside the openipc/ tree so builder.sh's # rm -rf doesn't touch it) and point Buildroot at it via BR2_DL_DIR # env var (honored per Buildroot manual section "Location of source # tarballs"). Mirrors OpenIPC/firmware/.github/workflows/build.yml's # output/dl cache, adapted for builder's clean-clone behavior. - name: Setup dl cache if: env.RUN uses: actions/cache@v4 with: path: /tmp/builder-dl key: builder-dl-${{env.CACHE_DATE}} restore-keys: builder-dl- - name: Refresh moving-ref package downloads if: env.RUN run: | # builder caches BR2_DL_DIR (/tmp/builder-dl) under a monthly key # (date +%m) and actions/cache only saves on a key miss, so the first build # of the month freezes it for weeks. Packages pinned to a moving ref keep a # constant download filename, so buildroot reuses the stale tarball — even # though builder re-clones firmware HEAD each run for freshness. Two forms: # - dash: -.tar.gz e.g. majestic-webui-dist.tar.gz # - dot: ..tar.bz2 e.g. majestic.t31.lite.master.tar.bz2 # The dot form is majestic's S3 tarball: it LOOKS S3-pinned but is a moving # `master` build re-uploaded every majestic CI run, with no .hash to force a # refetch — so it must be evicted too. (Empirically: a 06-14 nightly shipped a # ~2-week-old majestic-webui; later a t31 lite image shipped a pre-x-groups # majestic against the current webui — "No settings groups in schema".) Drop # both forms so buildroot re-fetches the current ref. Only genuinely # content-addressed tarballs (semver / SHA in the name) keep their cache. find /tmp/builder-dl -type f -regextype posix-extended \ -regex '.*[-.](HEAD|master|main|dist)\.tar\.(gz|bz2|xz)' \ -print -delete 2>/dev/null || true - name: Build firmware if: env.RUN run: | export GIT_HASH=$(git rev-parse --short ${GITHUB_SHA}) export GIT_BRANCH=${GITHUB_REF_NAME} echo GIT_HASH=${GIT_HASH} >> ${GITHUB_ENV} echo GIT_BRANCH=${GIT_BRANCH} >> ${GITHUB_ENV} mkdir -p /tmp/ccache ln -s /tmp/ccache ${HOME}/.ccache mkdir -p /tmp/builder-dl export BR2_DL_DIR=/tmp/builder-dl NAME=${{matrix.platform}} # Retry budget for transient upstream toolchain/CDN flakes # (matches OpenIPC/firmware/.github/workflows/build.yml). backoffs="30 60 120 300 600 1200" attempt=1 for sleep_for in $backoffs ""; do bash builder.sh ${NAME} && break if [ -z "$sleep_for" ]; then echo "::error::build failed after ${attempt} attempts" exit 1 fi echo "::warning::attempt ${attempt} failed, retrying after ${sleep_for}s" sleep "$sleep_for" attempt=$((attempt + 1)) done cd openipc TIME=$(date -d @${SECONDS} +%M:%S) echo TIME=${TIME} >> ${GITHUB_ENV} COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}') NORFW=$(find output/images -name openipc*nor*) if [ ! -z ${NORFW} ]; then if [ ${COMMON} -eq 1 ]; then echo NORFW=openipc/${NORFW} >> ${GITHUB_ENV} else mv ${NORFW} ../${NAME}-nor.tgz echo NORFW=${NAME}-nor.tgz >> ${GITHUB_ENV} fi fi NANDFW=$(find output/images -name openipc*nand*) if [ ! -z ${NANDFW} ]; then if [ ${COMMON} -eq 1 ]; then echo NANDFW=openipc/${NANDFW} >> ${GITHUB_ENV} else mv ${NANDFW} ../${NAME}-nand.tgz echo NANDFW=${NAME}-nand.tgz >> ${GITHUB_ENV} fi fi if [ -z ${NORFW} ] && [ -z ${NANDFW} ]; then exit 1 fi - name: Build size report if: env.NORFW || env.NANDFW run: | cd openipc NAME=${{matrix.platform}} make BOARD=${NAME} size-report || \ { echo "::warning::size-report failed for ${NAME}"; exit 0; } SIZES=$(find output/images -name 'sizes.*.json' | head -1) if [ -z "${SIZES}" ]; then exit 0 fi # Mirror the .tgz rename trick: compound matrix entries # (`__-`) share `_` and # would collide on the firmware-side `sizes.-.json` # filename when uploaded to the same release tag. Rename to # `sizes..json` so every compound device gets its own asset. COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}') if [ ${COMMON} -eq 1 ]; then echo "SIZES=openipc/${SIZES}" >> ${GITHUB_ENV} else mv ${SIZES} ../sizes.${NAME}.json echo "SIZES=sizes.${NAME}.json" >> ${GITHUB_ENV} fi # Stage everything this device produced into one flat directory, so the # artifact has a predictable shape no matter which of the two naming # schemes the build used (compound devices rename to -nor.tgz in # the workspace root, simple ones leave openipc.-nor-.tgz # under openipc/output/images/). - name: Stage artifacts if: >- github.event_name != 'pull_request' && (env.NORFW || env.NANDFW) run: | mkdir -p dist for f in "${NORFW:-}" "${NANDFW:-}" "${SIZES:-}"; do # An `if` rather than a `&&` chain: the step runs under `bash -e`, # where a trailing `&&` that tests false is only harmless because # something after it resets the status. if [ -n "$f" ] && [ -e "$f" ]; then cp -a "$f" dist/ fi done ls -la dist # Hand the images to the single `publish` job instead of writing releases # from here. 107 matrix jobs each deleting and re-uploading assets on the # SAME three shared releases is ~400 concurrent writes; OpenIPC/firmware # ran exactly that design and the releases API answered HTTP 500 under the # contention (run 27108181857: every board built, 14 jobs red in their # upload step). Consolidating every release write into one job removes the # concurrent-writer contention that causes it. - name: Upload build artifacts if: >- github.event_name != 'pull_request' && (env.NORFW || env.NANDFW) uses: actions/upload-artifact@v4 with: name: fw-${{ matrix.platform }} if-no-files-found: ignore retention-days: 1 path: dist/* - name: Send binary if: github.event_name != 'pull_request' && env.NORFW run: | TG_MSG="Build: ${BUILD_ID}\nCommit: ${GIT_HASH}\nBranch: ${GIT_BRANCH}\nTime: ${TIME}\n\n" TG_ICON="\xE2\x9C\x85 GitHub Actions" TG_HEADER=$(echo -e ${TG_MSG}${TG_ICON}) TG_TOKEN=${{secrets.TELEGRAM_TOKEN_BOT_OPENIPC}} TG_CHANNEL=${{secrets.TELEGRAM_CHANNEL_OPENIPC_DEV}} HTTP=$(curl -s -o /dev/null -w %{http_code} https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${NORFW}) echo Telegram response: ${HTTP} # All release writes happen here, once, so only a SINGLE job ever touches the # shared `nightly`/`latest` releases and the per-run dated one. Best-effort by # design: publish whatever the matrix produced even if some devices failed, # which matches enrich_manifest.py -- it indexes whatever assets exist. Never # runs on a pull request; there are no artifacts to publish there. publish: name: Publish releases needs: [preflight, select, buildroot] # The leading status function is load-bearing. A job whose `needs` failed is # skipped by default, and an `if:` lifts that only when it CONTAINS a status # function -- reading `needs.buildroot.result` inside the `contains()` below # does not count. So since #121 gave this job the matrix as a dependency, # any single failing device skipped it outright and the whole release went # unwritten: 2026-08-24 lost every device's nightly to one failure out of 98 # (ssc333_lite_meari-speed-6s), and 2026-08-21 lost it to two out of 98. # There is no nightly-20260821-* or nightly-20260824-* release as a result. # Before #121 each device uploaded from its own matrix job, so a bad device # took down only itself; this restores that without giving up the # single-writer fix that stopped the concurrent-upload rate limits. # # `!cancelled()` rather than `always()`: the `contains()` already drops a # cancelled or skipped matrix, but a run cancelled AFTER a green matrix # would still start uploading under `always()`, and a half-written nightly # is the thing this job exists to avoid. if: >- !cancelled() && github.event_name != 'pull_request' && needs.preflight.outputs.should_build == 'true' && contains(fromJSON('["success", "failure"]'), needs.buildroot.result) runs-on: ubuntu-latest permissions: contents: write steps: # continue-on-error only so that a matrix where every device failed -- # and therefore uploaded no fw-* artifact at all -- does not hard-fail # here before the gate can report why. Collect below turns that back into # a failure for every case except that one, so a download that breaks can # no longer publish nothing and call it success. - name: Download build artifacts id: download uses: actions/download-artifact@v4 continue-on-error: true with: pattern: fw-* path: dist merge-multiple: true - name: Collect assets id: collect run: | mkdir -p dist # tr because wc pads its output on some platforms (BSD does, the GNU # coreutils on the runner does not) and a padded " 0" would read # as non-empty to the `!= '0'` guard below. count=$(find dist -type f | wc -l | tr -d '[:space:]') echo "Collected ${count} asset(s):" ls -la dist || true echo "count=${count}" >> "$GITHUB_OUTPUT" # A matrix that went green must have produced artifacts. Zero assets # after that means the download failed, not that there was nothing to # publish -- and publishing nothing quietly is exactly how a nightly # goes missing while the run stays green. if [ "${{ needs.buildroot.result }}" = "success" ]; then if [ "${{ steps.download.outcome }}" != "success" ]; then echo "::error::artifact download failed after a fully successful matrix" exit 1 fi if [ "${count}" -eq 0 ]; then echo "::error::no artifacts collected although every device built" exit 1 fi elif [ "${count}" -eq 0 ]; then echo "::notice::matrix failed and produced nothing; no release to write" else # The partial case this job was always meant to serve, unreachable # until the `!cancelled()` above. Say so out loud: the devices that # did not build keep their PREVIOUS image on nightly/latest, which # is the right outcome for a flasher but is invisible next to the # freshly-written ones. echo "::warning::partial publish: matrix failed, writing the ${count} asset(s) that did build; nightly and latest keep their previous image for every device that did not build — see the failed jobs in this run for which" fi # Drive the release writes with gh rather than softprops/action-gh-release, # which uploads assets CONCURRENTLY with no throttle or retry knob. Firing # a few hundred asset writes at three releases trips GitHub's per-actor # secondary rate limit; here every asset goes up one at a time, paced under # the per-minute mutation ceiling, with exponential backoff so a transient # 403/429 is retried rather than failing the nightly. - name: Publish releases (paced, retry-aware) if: steps.collect.outputs.count != '0' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} BUILD_ID: ${{ needs.preflight.outputs.build_id }} HEAD_SHA: ${{ needs.preflight.outputs.head_sha }} SHORT_SHA: ${{ needs.preflight.outputs.short_sha }} BUILT_AT: ${{ needs.preflight.outputs.built_at }} run: | set -euo pipefail gh_retry() { local n=0 max=6 delay=15 until "$@"; do n=$((n + 1)) if [ "$n" -ge "$max" ]; then echo "::error::gh failed after ${max} attempts: $*" return 1 fi echo "::warning::gh attempt ${n} failed, sleeping ${delay}s before retry" sleep "$delay" delay=$((delay * 2)) done } ensure_release() { # [extra gh release create args...] local tag="$1"; shift if ! gh release view "$tag" >/dev/null 2>&1; then gh_retry gh release create "$tag" --target "$HEAD_SHA" --notes "$NOTES" "$@" fi } # One asset at a time, paced ~1/s to stay under the per-minute # content-mutation ceiling that triggers the secondary rate limit. upload_paced() { # ... local tag="$1"; shift local total=$# i=0 for f in "$@"; do i=$((i + 1)) echo "[${tag}] (${i}/${total}) ${f##*/}" gh_retry gh release upload "$tag" "$f" --clobber sleep 1 done } NOTES=$(printf 'sha=%s\nshort=%s\nbuilt_at=%s\n' "$HEAD_SHA" "$SHORT_SHA" "$BUILT_AT") # Full asset set (images + size sidecars) -> the dated release, which # is the only one enrich_manifest.py reads. mapfile -t DATED < <(find dist -maxdepth 1 -type f | sort) # nightly/latest are firmware-delivery aliases for flashers, so ship # only the images there. Re-uploading the size sidecars to two more # releases is pure rate-limit cost: the manifest never reads them from # these tags, and nothing else does either. mapfile -t IMAGES < <(find dist -maxdepth 1 -type f -name '*.tgz' | sort) # --- dated: immutable per-build history --- ensure_release "$BUILD_ID" --prerelease --title "$BUILD_ID" upload_paced "$BUILD_ID" "${DATED[@]}" # nightly and latest only ever move when there is firmware to move # them to. dist/ can hold sidecars and no images -- one device that # produced a size report and no .tgz is enough -- and pointing the two # tags users flash from at a build with no images behind them is worse # than leaving them on yesterday's. if [ "${#IMAGES[@]}" -eq 0 ]; then echo "::warning::no images in this build; leaving nightly and latest where they are" else # --- rolling nightly: move tag + body to this build, refresh images --- ensure_release nightly gh_retry gh release edit nightly --notes "$NOTES" gh_retry gh api -X PATCH "repos/${GH_REPO}/git/refs/tags/nightly" -f sha="$HEAD_SHA" -F force=true upload_paced nightly "${IMAGES[@]}" # --- latest: legacy alias --- ensure_release latest gh_retry gh api -X PATCH "repos/${GH_REPO}/git/refs/tags/latest" -f sha="$HEAD_SHA" -F force=true upload_paced latest "${IMAGES[@]}" fi # Single umbrella status check covering the whole run, so branch protection # can require one context instead of a hardcoded "Build ()" per # device -- the matrix is now dynamic, and its job names change per PR. # There was no gate here at all before: nothing reported a verdict on a # change, because nothing built one. ci-gate: name: CI Gate needs: [select, preflight, buildroot, publish] # always() so the gate still reports when a needed job fails -- but not on # a run the guard above deliberately skipped, where it would read # select=skipped and turn a saved build into a red run. if: >- always() && (github.repository == 'OpenIPC/builder' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && !github.event.repository.private)) runs-on: ubuntu-latest steps: - name: Require selection + device matrix to succeed run: | echo "select=${{ needs.select.result }} preflight=${{ needs.preflight.result }} buildroot=${{ needs.buildroot.result }} publish=${{ needs.publish.result }}" if [ "${{ needs.select.result }}" != "success" ]; then echo "::error::device selection did not succeed"; exit 1 fi if [ "${{ needs.preflight.result }}" != "success" ]; then echo "::error::preflight did not succeed"; exit 1 fi # Check the matrix against what select asked for. Blanket-accepting # 'skipped' would mean a selector bug that narrowed a real change # down to nothing reads as a green PR. if [ "${{ needs.select.outputs.needs-build }}" = "true" ] && \ [ "${{ needs.preflight.outputs.should_build }}" = "true" ]; then if [ "${{ needs.buildroot.result }}" != "success" ]; then echo "::error::device matrix result=${{ needs.buildroot.result }}"; exit 1 fi echo "device matrix OK (success)" else # Nothing in the change reaches a build, or the PR is still a draft. if [ "${{ needs.buildroot.result }}" != "skipped" ]; then echo "::error::device matrix ran when it should not have (${{ needs.buildroot.result }})"; exit 1 fi echo "device matrix OK (no devices to build)" fi # publish is skipped on PRs and on a run that built nothing; only a # real failure of the single release-writer should fail the gate. case "${{ needs.publish.result }}" in success|skipped) echo "publish OK (${{ needs.publish.result }})" ;; *) echo "::error::publish result=${{ needs.publish.result }}"; exit 1 ;; esac