openipc-builder/.github/workflows/master.yml

name: Build
on:
  # There was no pull_request trigger here: a change landed unbuilt and the
  # nightly found out the next morning, or a user did. Building all 107 devices
  # per PR is what made that unaffordable, so .github/scripts/ci-matrix.py picks
  # only the devices a PR can reach. ready_for_review is not in the default set
  # (opened, synchronize, reopened), and the selector skips drafts, so without it
  # a PR opened as a draft would never build once marked ready.
  pull_request:
    branches:
      - master
    types: [opened, synchronize, reopened, ready_for_review]
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:
    inputs:
      platform:
        description: Platform
        required: false

# Cancel a PR's earlier run when a new commit lands on it. Scheduled and
# dispatched runs are deliberately left alone: they write releases, and a
# cancelled publish leaves the nightly half-uploaded.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  # Which devices can this change actually reach? Only ever narrows a
  # pull_request -- the nightly is decided by firmware HEAD, not by this repo's
  # diff, so every other event gets the whole matrix.
  #
  # The narrowing happens HERE, in a job that always runs, and never in a
  # `paths:` filter on the trigger: a filtered-out workflow does not run at all,
  # so a required check would stay pending forever and the PR could never merge.
  select:
    name: Select devices
    # Same guard as preflight below.
    if: >-
      github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private)
    runs-on: ubuntu-latest
    outputs:
      matrix:      ${{ steps.pick.outputs.matrix }}
      needs-build: ${{ steps.pick.outputs.needs-build }}
    steps:
      - uses: actions/checkout@v4
      - name: Check the selector still describes the tree
        run: python3 .github/scripts/ci-matrix.py --self-test
      - id: pick
        name: Pick the affected devices
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: python3 .github/scripts/ci-matrix.py

  preflight:
    name: Preflight
    # Guard rail for clones of this repo. A clone pushed to a new repository
    # (a private mirror, an internal re-host) inherits this file along with
    # its cron, and the full platform matrix then runs on that owner's Actions
    # bill, nightly and unattended, publishing nightlies nobody reads. Forks
    # are partially covered (GitHub disables scheduled workflows there), a
    # fresh repository is not. Mirrors OpenIPC/firmware#2255.
    #
    # Unattended runs happen only on the canonical repo. Anywhere else a
    # workflow runs only if someone explicitly asked for it, or if it is both
    # free and attended -- which is what the pull_request clause covers, now
    # that this workflow has such a trigger.
    if: >-
      github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private)
    runs-on: ubuntu-latest
    outputs:
      should_build: ${{ steps.gate.outputs.should_build }}
      head_sha:     ${{ steps.gate.outputs.head_sha }}
      short_sha:    ${{ steps.gate.outputs.short_sha }}
      build_id:     ${{ steps.gate.outputs.build_id }}
      built_at:     ${{ steps.gate.outputs.built_at }}
    steps:
      - uses: actions/checkout@v4
      - id: gate
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          HEAD=$(git rev-parse HEAD)
          SHORT=$(git rev-parse --short HEAD)
          BUILD_ID="nightly-$(date -u +%Y%m%d)-${SHORT}"
          BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
          # Nightly rebuilds unconditionally on schedule. builder.sh re-clones
          # OpenIPC/firmware at HEAD on every run, so the relevant inputs are
          # mostly outside this repo (firmware HEAD, toolchain release, kernel
          # tarball, vendor osdrv, individual package upstreams). The previous
          # gate skipped when only this repo's HEAD matched the last published
          # nightly, which made upstream fixes (e.g. an openhisilicon mipi_rx
          # regression fix that lands in firmware) invisible to users until
          # something here happened to change. Always build on schedule.
          echo "Build: $BUILD_ID (event=${{ github.event_name }})"
          echo "should_build=true"  >> "$GITHUB_OUTPUT"
          echo "head_sha=$HEAD"      >> "$GITHUB_OUTPUT"
          echo "short_sha=$SHORT"    >> "$GITHUB_OUTPUT"
          echo "build_id=$BUILD_ID"  >> "$GITHUB_OUTPUT"
          echo "built_at=$BUILT_AT"  >> "$GITHUB_OUTPUT"

  buildroot:
    name: Firmware
    needs: [preflight, select]
    if: >-
      needs.preflight.outputs.should_build == 'true' &&
      needs.select.outputs.needs-build == 'true'
    runs-on: ubuntu-latest
    env:
      BUILD_ID:       ${{ needs.preflight.outputs.build_id }}
      BUILD_SHA:      ${{ needs.preflight.outputs.head_sha }}
      BUILD_PLATFORM: ${{ matrix.platform }}

    strategy:
      fail-fast: false
      # The device list lives in .github/scripts/ci-matrix.py, which is
      # also what narrows it. Everything except pull_request gets all of it.
      matrix: ${{ fromJson(needs.select.outputs.matrix) }}
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Prepare firmware
        run: |
          echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf
          echo CACHE_DATE=$(date +%m) >> ${GITHUB_ENV}
          INPUT=${{inputs.platform}}
          MATRIX=$(echo ${{matrix.platform}} | cut -d_ -f1)
          if [ -z ${INPUT} ] || [ ${INPUT} = ${MATRIX} ]; then
            echo RUN=${MATRIX} >> ${GITHUB_ENV}
          fi

      - name: Setup ccache
        if: env.RUN
        uses: actions/cache@v4
        with:
          path: /tmp/ccache
          key: ${{matrix.platform}}-${{env.CACHE_DATE}}

      # builder.sh does `rm -rf openipc` at the top of every run to force a
      # clean re-clone of OpenIPC/firmware, which also nukes the default
      # `openipc/output/dl/` directory Buildroot uses for downloaded source
      # tarballs. Without a cache, every cron run re-downloads ~1-3 GB of
      # tarballs from upstream mirrors — and any one of them randomly
      # failing (e.g. cgit `snapshot/` URLs that regenerate tarballs per
      # request with shifting sha256, like wireguard-tools on
      # git.zx2c4.com) takes down the matrix entry for that board. See
      # kaeru wireguard-zx2c4-tarball-regeneration-hash-drift for the
      # specific incident this caching addresses.
      #
      # Cache to /tmp/builder-dl (outside the openipc/ tree so builder.sh's
      # rm -rf doesn't touch it) and point Buildroot at it via BR2_DL_DIR
      # env var (honored per Buildroot manual section "Location of source
      # tarballs"). Mirrors OpenIPC/firmware/.github/workflows/build.yml's
      # output/dl cache, adapted for builder's clean-clone behavior.
      - name: Setup dl cache
        if: env.RUN
        uses: actions/cache@v4
        with:
          path: /tmp/builder-dl
          key: builder-dl-${{env.CACHE_DATE}}
          restore-keys: builder-dl-

      - name: Refresh moving-ref package downloads
        if: env.RUN
        run: |
          # builder caches BR2_DL_DIR (/tmp/builder-dl) under a monthly key
          # (date +%m) and actions/cache only saves on a key miss, so the first build
          # of the month freezes it for weeks. Packages pinned to a moving ref keep a
          # constant download filename, so buildroot reuses the stale tarball — even
          # though builder re-clones firmware HEAD each run for freshness. Two forms:
          #   - dash:  <pkg>-<ref>.tar.gz   e.g. majestic-webui-dist.tar.gz
          #   - dot:   <pkg>.<ref>.tar.bz2  e.g. majestic.t31.lite.master.tar.bz2
          # The dot form is majestic's S3 tarball: it LOOKS S3-pinned but is a moving
          # `master` build re-uploaded every majestic CI run, with no .hash to force a
          # refetch — so it must be evicted too. (Empirically: a 06-14 nightly shipped a
          # ~2-week-old majestic-webui; later a t31 lite image shipped a pre-x-groups
          # majestic against the current webui — "No settings groups in schema".) Drop
          # both forms so buildroot re-fetches the current ref. Only genuinely
          # content-addressed tarballs (semver / SHA in the name) keep their cache.
          find /tmp/builder-dl -type f -regextype posix-extended \
            -regex '.*[-.](HEAD|master|main|dist)\.tar\.(gz|bz2|xz)' \
            -print -delete 2>/dev/null || true

      - name: Build firmware
        if: env.RUN
        run: |
          export GIT_HASH=$(git rev-parse --short ${GITHUB_SHA})
          export GIT_BRANCH=${GITHUB_REF_NAME}
          echo GIT_HASH=${GIT_HASH} >> ${GITHUB_ENV}
          echo GIT_BRANCH=${GIT_BRANCH} >> ${GITHUB_ENV}

          mkdir -p /tmp/ccache
          ln -s /tmp/ccache ${HOME}/.ccache

          mkdir -p /tmp/builder-dl
          export BR2_DL_DIR=/tmp/builder-dl

          NAME=${{matrix.platform}}
          # Retry budget for transient upstream toolchain/CDN flakes
          # (matches OpenIPC/firmware/.github/workflows/build.yml).
          backoffs="30 60 120 300 600 1200"
          attempt=1
          for sleep_for in $backoffs ""; do
            bash builder.sh ${NAME} && break
            if [ -z "$sleep_for" ]; then
              echo "::error::build failed after ${attempt} attempts"
              exit 1
            fi
            echo "::warning::attempt ${attempt} failed, retrying after ${sleep_for}s"
            sleep "$sleep_for"
            attempt=$((attempt + 1))
          done
          cd openipc

          TIME=$(date -d @${SECONDS} +%M:%S)
          echo TIME=${TIME} >> ${GITHUB_ENV}
          COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}')

          NORFW=$(find output/images -name openipc*nor*)
          if [ ! -z ${NORFW} ]; then
            if [ ${COMMON} -eq 1 ]; then
              echo NORFW=openipc/${NORFW} >> ${GITHUB_ENV}
            else
              mv ${NORFW} ../${NAME}-nor.tgz
              echo NORFW=${NAME}-nor.tgz >> ${GITHUB_ENV}
            fi
          fi

          NANDFW=$(find output/images -name openipc*nand*)
          if [ ! -z ${NANDFW} ]; then
            if [ ${COMMON} -eq 1 ]; then
              echo NANDFW=openipc/${NANDFW} >> ${GITHUB_ENV}
            else
              mv ${NANDFW} ../${NAME}-nand.tgz
              echo NANDFW=${NAME}-nand.tgz >> ${GITHUB_ENV}
            fi
          fi

          if [ -z ${NORFW} ] && [ -z ${NANDFW} ]; then
            exit 1
          fi

      - name: Build size report
        if: env.NORFW || env.NANDFW
        run: |
          cd openipc
          NAME=${{matrix.platform}}
          make BOARD=${NAME} size-report || \
            { echo "::warning::size-report failed for ${NAME}"; exit 0; }
          SIZES=$(find output/images -name 'sizes.*.json' | head -1)
          if [ -z "${SIZES}" ]; then
            exit 0
          fi
          # Mirror the .tgz rename trick: compound matrix entries
          # (`<soc>_<variant>_<vendor>-<model>`) share `<soc>_<variant>` and
          # would collide on the firmware-side `sizes.<soc>-<variant>.json`
          # filename when uploaded to the same release tag. Rename to
          # `sizes.<NAME>.json` so every compound device gets its own asset.
          COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}')
          if [ ${COMMON} -eq 1 ]; then
            echo "SIZES=openipc/${SIZES}" >> ${GITHUB_ENV}
          else
            mv ${SIZES} ../sizes.${NAME}.json
            echo "SIZES=sizes.${NAME}.json" >> ${GITHUB_ENV}
          fi

      # Stage everything this device produced into one flat directory, so the
      # artifact has a predictable shape no matter which of the two naming
      # schemes the build used (compound devices rename to <device>-nor.tgz in
      # the workspace root, simple ones leave openipc.<soc>-nor-<variant>.tgz
      # under openipc/output/images/).
      - name: Stage artifacts
        if: >-
          github.event_name != 'pull_request' &&
          (env.NORFW || env.NANDFW)
        run: |
          mkdir -p dist
          for f in "${NORFW:-}" "${NANDFW:-}" "${SIZES:-}"; do
            # An `if` rather than a `&&` chain: the step runs under `bash -e`,
            # where a trailing `&&` that tests false is only harmless because
            # something after it resets the status.
            if [ -n "$f" ] && [ -e "$f" ]; then
              cp -a "$f" dist/
            fi
          done
          ls -la dist

      # Hand the images to the single `publish` job instead of writing releases
      # from here. 107 matrix jobs each deleting and re-uploading assets on the
      # SAME three shared releases is ~400 concurrent writes; OpenIPC/firmware
      # ran exactly that design and the releases API answered HTTP 500 under the
      # contention (run 27108181857: every board built, 14 jobs red in their
      # upload step). Consolidating every release write into one job removes the
      # concurrent-writer contention that causes it.
      - name: Upload build artifacts
        if: >-
          github.event_name != 'pull_request' &&
          (env.NORFW || env.NANDFW)
        uses: actions/upload-artifact@v4
        with:
          name: fw-${{ matrix.platform }}
          if-no-files-found: ignore
          retention-days: 1
          path: dist/*

      - name: Send binary
        if: github.event_name != 'pull_request' && env.NORFW
        run: |
          TG_MSG="Build: ${BUILD_ID}\nCommit: ${GIT_HASH}\nBranch: ${GIT_BRANCH}\nTime: ${TIME}\n\n"
          TG_ICON="\xE2\x9C\x85 GitHub Actions"
          TG_HEADER=$(echo -e ${TG_MSG}${TG_ICON})
          TG_TOKEN=${{secrets.TELEGRAM_TOKEN_BOT_OPENIPC}}
          TG_CHANNEL=${{secrets.TELEGRAM_CHANNEL_OPENIPC_DEV}}
          HTTP=$(curl -s -o /dev/null -w %{http_code} https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${NORFW})
          echo Telegram response: ${HTTP}

  # All release writes happen here, once, so only a SINGLE job ever touches the
  # shared `nightly`/`latest` releases and the per-run dated one. Best-effort by
  # design: publish whatever the matrix produced even if some devices failed,
  # which matches enrich_manifest.py -- it indexes whatever assets exist. Never
  # runs on a pull request; there are no artifacts to publish there.
  publish:
    name: Publish releases
    needs: [preflight, select, buildroot]
    if: >-
      github.event_name != 'pull_request' &&
      needs.preflight.outputs.should_build == 'true' &&
      contains(fromJSON('["success", "failure"]'), needs.buildroot.result)
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      # continue-on-error only so that a matrix where every device failed --
      # and therefore uploaded no fw-* artifact at all -- does not hard-fail
      # here before the gate can report why. Collect below turns that back into
      # a failure for every case except that one, so a download that breaks can
      # no longer publish nothing and call it success.
      - name: Download build artifacts
        id: download
        uses: actions/download-artifact@v4
        continue-on-error: true
        with:
          pattern: fw-*
          path: dist
          merge-multiple: true

      - name: Collect assets
        id: collect
        run: |
          mkdir -p dist
          # tr because wc pads its output on some platforms (BSD does, the GNU
          # coreutils on the runner does not) and a padded "     0" would read
          # as non-empty to the `!= '0'` guard below.
          count=$(find dist -type f | wc -l | tr -d '[:space:]')
          echo "Collected ${count} asset(s):"
          ls -la dist || true
          echo "count=${count}" >> "$GITHUB_OUTPUT"

          # A matrix that went green must have produced artifacts. Zero assets
          # after that means the download failed, not that there was nothing to
          # publish -- and publishing nothing quietly is exactly how a nightly
          # goes missing while the run stays green.
          if [ "${{ needs.buildroot.result }}" = "success" ]; then
            if [ "${{ steps.download.outcome }}" != "success" ]; then
              echo "::error::artifact download failed after a fully successful matrix"
              exit 1
            fi
            if [ "${count}" -eq 0 ]; then
              echo "::error::no artifacts collected although every device built"
              exit 1
            fi
          elif [ "${count}" -eq 0 ]; then
            echo "::notice::matrix failed and produced nothing; no release to write"

      # Drive the release writes with gh rather than softprops/action-gh-release,
      # which uploads assets CONCURRENTLY with no throttle or retry knob. Firing
      # a few hundred asset writes at three releases trips GitHub's per-actor
      # secondary rate limit; here every asset goes up one at a time, paced under
      # the per-minute mutation ceiling, with exponential backoff so a transient
      # 403/429 is retried rather than failing the nightly.
      - name: Publish releases (paced, retry-aware)
        if: steps.collect.outputs.count != '0'
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_REPO: ${{ github.repository }}
          BUILD_ID:  ${{ needs.preflight.outputs.build_id }}
          HEAD_SHA:  ${{ needs.preflight.outputs.head_sha }}
          SHORT_SHA: ${{ needs.preflight.outputs.short_sha }}
          BUILT_AT:  ${{ needs.preflight.outputs.built_at }}
        run: |
          set -euo pipefail

          gh_retry() {
            local n=0 max=6 delay=15
            until "$@"; do
              n=$((n + 1))
              if [ "$n" -ge "$max" ]; then
                echo "::error::gh failed after ${max} attempts: $*"
                return 1
              fi
              echo "::warning::gh attempt ${n} failed, sleeping ${delay}s before retry"
              sleep "$delay"
              delay=$((delay * 2))
            done
          }

          ensure_release() { # <tag> [extra gh release create args...]
            local tag="$1"; shift
            if ! gh release view "$tag" >/dev/null 2>&1; then
              gh_retry gh release create "$tag" --target "$HEAD_SHA" --notes "$NOTES" "$@"
            fi
          }

          # One asset at a time, paced ~1/s to stay under the per-minute
          # content-mutation ceiling that triggers the secondary rate limit.
          upload_paced() { # <tag> <file>...
            local tag="$1"; shift
            local total=$# i=0
            for f in "$@"; do
              i=$((i + 1))
              echo "[${tag}] (${i}/${total}) ${f##*/}"
              gh_retry gh release upload "$tag" "$f" --clobber
              sleep 1
            done
          }

          NOTES=$(printf 'sha=%s\nshort=%s\nbuilt_at=%s\n' "$HEAD_SHA" "$SHORT_SHA" "$BUILT_AT")

          # Full asset set (images + size sidecars) -> the dated release, which
          # is the only one enrich_manifest.py reads.
          mapfile -t DATED < <(find dist -maxdepth 1 -type f | sort)
          # nightly/latest are firmware-delivery aliases for flashers, so ship
          # only the images there. Re-uploading the size sidecars to two more
          # releases is pure rate-limit cost: the manifest never reads them from
          # these tags, and nothing else does either.
          mapfile -t IMAGES < <(find dist -maxdepth 1 -type f -name '*.tgz' | sort)

          # --- dated: immutable per-build history ---
          ensure_release "$BUILD_ID" --prerelease --title "$BUILD_ID"
          upload_paced "$BUILD_ID" "${DATED[@]}"

          # nightly and latest only ever move when there is firmware to move
          # them to. dist/ can hold sidecars and no images -- one device that
          # produced a size report and no .tgz is enough -- and pointing the two
          # tags users flash from at a build with no images behind them is worse
          # than leaving them on yesterday's.
          if [ "${#IMAGES[@]}" -eq 0 ]; then
            echo "::warning::no images in this build; leaving nightly and latest where they are"
          else
            # --- rolling nightly: move tag + body to this build, refresh images ---
            ensure_release nightly
            gh_retry gh release edit nightly --notes "$NOTES"
            gh_retry gh api -X PATCH "repos/${GH_REPO}/git/refs/tags/nightly" -f sha="$HEAD_SHA" -F force=true
            upload_paced nightly "${IMAGES[@]}"

            # --- latest: legacy alias ---
            ensure_release latest
            gh_retry gh api -X PATCH "repos/${GH_REPO}/git/refs/tags/latest" -f sha="$HEAD_SHA" -F force=true
            upload_paced latest "${IMAGES[@]}"
          fi

  # Single umbrella status check covering the whole run, so branch protection
  # can require one context instead of a hardcoded "Build (<device>)" per
  # device -- the matrix is now dynamic, and its job names change per PR.
  # There was no gate here at all before: nothing reported a verdict on a
  # change, because nothing built one.
  ci-gate:
    name: CI Gate
    needs: [select, preflight, buildroot, publish]
    # always() so the gate still reports when a needed job fails -- but not on
    # a run the guard above deliberately skipped, where it would read
    # select=skipped and turn a saved build into a red run.
    if: >-
      always() && (github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private))
    runs-on: ubuntu-latest
    steps:
      - name: Require selection + device matrix to succeed
        run: |
          echo "select=${{ needs.select.result }} preflight=${{ needs.preflight.result }} buildroot=${{ needs.buildroot.result }} publish=${{ needs.publish.result }}"
          if [ "${{ needs.select.result }}" != "success" ]; then
            echo "::error::device selection did not succeed"; exit 1
          fi
          if [ "${{ needs.preflight.result }}" != "success" ]; then
            echo "::error::preflight did not succeed"; exit 1
          fi
          # Check the matrix against what select asked for. Blanket-accepting
          # 'skipped' would mean a selector bug that narrowed a real change
          # down to nothing reads as a green PR.
          if [ "${{ needs.select.outputs.needs-build }}" = "true" ] && \
             [ "${{ needs.preflight.outputs.should_build }}" = "true" ]; then
            if [ "${{ needs.buildroot.result }}" != "success" ]; then
              echo "::error::device matrix result=${{ needs.buildroot.result }}"; exit 1
            fi
            echo "device matrix OK (success)"
          else
            # Nothing in the change reaches a build, or the PR is still a draft.
            if [ "${{ needs.buildroot.result }}" != "skipped" ]; then
              echo "::error::device matrix ran when it should not have (${{ needs.buildroot.result }})"; exit 1
            fi
            echo "device matrix OK (no devices to build)"
          fi
          # publish is skipped on PRs and on a run that built nothing; only a
          # real failure of the single release-writer should fail the gate.
          case "${{ needs.publish.result }}" in
            success|skipped) echo "publish OK (${{ needs.publish.result }})" ;;
            *) echo "::error::publish result=${{ needs.publish.result }}"; exit 1 ;;
          esac