openipc-builder/.github/workflows/master.yml

name: Build
on:
  # There was no pull_request trigger here: a change landed unbuilt and the
  # nightly found out the next morning, or a user did. Building all 107 devices
  # per PR is what made that unaffordable, so .github/scripts/ci-matrix.py picks
  # only the devices a PR can reach. ready_for_review is not in the default set
  # (opened, synchronize, reopened), and the selector skips drafts, so without it
  # a PR opened as a draft would never build once marked ready.
  pull_request:
    branches:
      - master
    types: [opened, synchronize, reopened, ready_for_review]
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:
    inputs:
      platform:
        description: Platform
        required: false

# Cancel a PR's earlier run when a new commit lands on it. Scheduled and
# dispatched runs are deliberately left alone: they write releases, and a
# cancelled publish leaves the nightly half-uploaded.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  # Which devices can this change actually reach? Only ever narrows a
  # pull_request -- the nightly is decided by firmware HEAD, not by this repo's
  # diff, so every other event gets the whole matrix.
  #
  # The narrowing happens HERE, in a job that always runs, and never in a
  # `paths:` filter on the trigger: a filtered-out workflow does not run at all,
  # so a required check would stay pending forever and the PR could never merge.
  select:
    name: Select devices
    # Same guard as preflight below.
    if: >-
      github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private)
    runs-on: ubuntu-latest
    outputs:
      matrix:      ${{ steps.pick.outputs.matrix }}
      needs-build: ${{ steps.pick.outputs.needs-build }}
    steps:
      - uses: actions/checkout@v4
      - name: Check the selector still describes the tree
        run: python3 .github/scripts/ci-matrix.py --self-test
      - id: pick
        name: Pick the affected devices
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: python3 .github/scripts/ci-matrix.py

  preflight:
    name: Preflight
    # Guard rail for clones of this repo. A clone pushed to a new repository
    # (a private mirror, an internal re-host) inherits this file along with
    # its cron, and the full platform matrix then runs on that owner's Actions
    # bill, nightly and unattended, publishing nightlies nobody reads. Forks
    # are partially covered (GitHub disables scheduled workflows there), a
    # fresh repository is not. Mirrors OpenIPC/firmware#2255.
    #
    # Unattended runs happen only on the canonical repo. Anywhere else a
    # workflow runs only if someone explicitly asked for it, or if it is both
    # free and attended -- which is what the pull_request clause covers, now
    # that this workflow has such a trigger.
    if: >-
      github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private)
    runs-on: ubuntu-latest
    outputs:
      should_build: ${{ steps.gate.outputs.should_build }}
      head_sha:     ${{ steps.gate.outputs.head_sha }}
      short_sha:    ${{ steps.gate.outputs.short_sha }}
      build_id:     ${{ steps.gate.outputs.build_id }}
      built_at:     ${{ steps.gate.outputs.built_at }}
    steps:
      - uses: actions/checkout@v4
      - id: gate
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          HEAD=$(git rev-parse HEAD)
          SHORT=$(git rev-parse --short HEAD)
          BUILD_ID="nightly-$(date -u +%Y%m%d)-${SHORT}"
          BUILT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
          # Nightly rebuilds unconditionally on schedule. builder.sh re-clones
          # OpenIPC/firmware at HEAD on every run, so the relevant inputs are
          # mostly outside this repo (firmware HEAD, toolchain release, kernel
          # tarball, vendor osdrv, individual package upstreams). The previous
          # gate skipped when only this repo's HEAD matched the last published
          # nightly, which made upstream fixes (e.g. an openhisilicon mipi_rx
          # regression fix that lands in firmware) invisible to users until
          # something here happened to change. Always build on schedule.
          echo "Build: $BUILD_ID (event=${{ github.event_name }})"
          echo "should_build=true"  >> "$GITHUB_OUTPUT"
          echo "head_sha=$HEAD"      >> "$GITHUB_OUTPUT"
          echo "short_sha=$SHORT"    >> "$GITHUB_OUTPUT"
          echo "build_id=$BUILD_ID"  >> "$GITHUB_OUTPUT"
          echo "built_at=$BUILT_AT"  >> "$GITHUB_OUTPUT"

  buildroot:
    name: Firmware
    needs: [preflight, select]
    if: >-
      needs.preflight.outputs.should_build == 'true' &&
      needs.select.outputs.needs-build == 'true'
    runs-on: ubuntu-latest
    env:
      BUILD_ID:       ${{ needs.preflight.outputs.build_id }}
      BUILD_SHA:      ${{ needs.preflight.outputs.head_sha }}
      BUILD_PLATFORM: ${{ matrix.platform }}

    strategy:
      fail-fast: false
      # The device list lives in .github/scripts/ci-matrix.py, which is
      # also what narrows it. Everything except pull_request gets all of it.
      matrix: ${{ fromJson(needs.select.outputs.matrix) }}
    steps:
      - name: Checkout source
        uses: actions/checkout@v4

      - name: Prepare firmware
        run: |
          echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf
          echo CACHE_DATE=$(date +%m) >> ${GITHUB_ENV}
          INPUT=${{inputs.platform}}
          MATRIX=$(echo ${{matrix.platform}} | cut -d_ -f1)
          if [ -z ${INPUT} ] || [ ${INPUT} = ${MATRIX} ]; then
            echo RUN=${MATRIX} >> ${GITHUB_ENV}
          fi

      - name: Setup ccache
        if: env.RUN
        uses: actions/cache@v4
        with:
          path: /tmp/ccache
          key: ${{matrix.platform}}-${{env.CACHE_DATE}}

      # builder.sh does `rm -rf openipc` at the top of every run to force a
      # clean re-clone of OpenIPC/firmware, which also nukes the default
      # `openipc/output/dl/` directory Buildroot uses for downloaded source
      # tarballs. Without a cache, every cron run re-downloads ~1-3 GB of
      # tarballs from upstream mirrors — and any one of them randomly
      # failing (e.g. cgit `snapshot/` URLs that regenerate tarballs per
      # request with shifting sha256, like wireguard-tools on
      # git.zx2c4.com) takes down the matrix entry for that board. See
      # kaeru wireguard-zx2c4-tarball-regeneration-hash-drift for the
      # specific incident this caching addresses.
      #
      # Cache to /tmp/builder-dl (outside the openipc/ tree so builder.sh's
      # rm -rf doesn't touch it) and point Buildroot at it via BR2_DL_DIR
      # env var (honored per Buildroot manual section "Location of source
      # tarballs"). Mirrors OpenIPC/firmware/.github/workflows/build.yml's
      # output/dl cache, adapted for builder's clean-clone behavior.
      - name: Setup dl cache
        if: env.RUN
        uses: actions/cache@v4
        with:
          path: /tmp/builder-dl
          key: builder-dl-${{env.CACHE_DATE}}
          restore-keys: builder-dl-

      - name: Refresh moving-ref package downloads
        if: env.RUN
        run: |
          # builder caches BR2_DL_DIR (/tmp/builder-dl) under a monthly key
          # (date +%m) and actions/cache only saves on a key miss, so the first build
          # of the month freezes it for weeks. Packages pinned to a moving ref keep a
          # constant download filename, so buildroot reuses the stale tarball — even
          # though builder re-clones firmware HEAD each run for freshness. Two forms:
          #   - dash:  <pkg>-<ref>.tar.gz   e.g. majestic-webui-dist.tar.gz
          #   - dot:   <pkg>.<ref>.tar.bz2  e.g. majestic.t31.lite.master.tar.bz2
          # The dot form is majestic's S3 tarball: it LOOKS S3-pinned but is a moving
          # `master` build re-uploaded every majestic CI run, with no .hash to force a
          # refetch — so it must be evicted too. (Empirically: a 06-14 nightly shipped a
          # ~2-week-old majestic-webui; later a t31 lite image shipped a pre-x-groups
          # majestic against the current webui — "No settings groups in schema".) Drop
          # both forms so buildroot re-fetches the current ref. Only genuinely
          # content-addressed tarballs (semver / SHA in the name) keep their cache.
          find /tmp/builder-dl -type f -regextype posix-extended \
            -regex '.*[-.](HEAD|master|main|dist)\.tar\.(gz|bz2|xz)' \
            -print -delete 2>/dev/null || true

      - name: Build firmware
        if: env.RUN
        run: |
          export GIT_HASH=$(git rev-parse --short ${GITHUB_SHA})
          export GIT_BRANCH=${GITHUB_REF_NAME}
          echo GIT_HASH=${GIT_HASH} >> ${GITHUB_ENV}
          echo GIT_BRANCH=${GIT_BRANCH} >> ${GITHUB_ENV}

          mkdir -p /tmp/ccache
          ln -s /tmp/ccache ${HOME}/.ccache

          mkdir -p /tmp/builder-dl
          export BR2_DL_DIR=/tmp/builder-dl

          NAME=${{matrix.platform}}
          # Retry budget for transient upstream toolchain/CDN flakes
          # (matches OpenIPC/firmware/.github/workflows/build.yml).
          backoffs="30 60 120 300 600 1200"
          attempt=1
          for sleep_for in $backoffs ""; do
            bash builder.sh ${NAME} && break
            if [ -z "$sleep_for" ]; then
              echo "::error::build failed after ${attempt} attempts"
              exit 1
            fi
            echo "::warning::attempt ${attempt} failed, retrying after ${sleep_for}s"
            sleep "$sleep_for"
            attempt=$((attempt + 1))
          done
          cd openipc

          TIME=$(date -d @${SECONDS} +%M:%S)
          echo TIME=${TIME} >> ${GITHUB_ENV}
          COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}')

          NORFW=$(find output/images -name openipc*nor*)
          if [ ! -z ${NORFW} ]; then
            if [ ${COMMON} -eq 1 ]; then
              echo NORFW=openipc/${NORFW} >> ${GITHUB_ENV}
            else
              mv ${NORFW} ../${NAME}-nor.tgz
              echo NORFW=${NAME}-nor.tgz >> ${GITHUB_ENV}
            fi
          fi

          NANDFW=$(find output/images -name openipc*nand*)
          if [ ! -z ${NANDFW} ]; then
            if [ ${COMMON} -eq 1 ]; then
              echo NANDFW=openipc/${NANDFW} >> ${GITHUB_ENV}
            else
              mv ${NANDFW} ../${NAME}-nand.tgz
              echo NANDFW=${NAME}-nand.tgz >> ${GITHUB_ENV}
            fi
          fi

          if [ -z ${NORFW} ] && [ -z ${NANDFW} ]; then
            exit 1
          fi

      - name: Build size report
        if: env.NORFW || env.NANDFW
        run: |
          cd openipc
          NAME=${{matrix.platform}}
          make BOARD=${NAME} size-report || \
            { echo "::warning::size-report failed for ${NAME}"; exit 0; }
          SIZES=$(find output/images -name 'sizes.*.json' | head -1)
          if [ -z "${SIZES}" ]; then
            exit 0
          fi
          # Mirror the .tgz rename trick: compound matrix entries
          # (`<soc>_<variant>_<vendor>-<model>`) share `<soc>_<variant>` and
          # would collide on the firmware-side `sizes.<soc>-<variant>.json`
          # filename when uploaded to the same release tag. Rename to
          # `sizes.<NAME>.json` so every compound device gets its own asset.
          COMMON=$(echo ${NAME} | awk -F_ '{print NF-1}')
          if [ ${COMMON} -eq 1 ]; then
            echo "SIZES=openipc/${SIZES}" >> ${GITHUB_ENV}
          else
            mv ${SIZES} ../sizes.${NAME}.json
            echo "SIZES=sizes.${NAME}.json" >> ${GITHUB_ENV}
          fi

      - name: Upload firmware (dated)
        if: >-
          github.event_name != 'pull_request' &&
          (env.NORFW || env.NANDFW)
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ needs.preflight.outputs.build_id }}
          prerelease: true
          body: |
            sha=${{ needs.preflight.outputs.head_sha }}
            short=${{ needs.preflight.outputs.short_sha }}
            built_at=${{ needs.preflight.outputs.built_at }}
          files: |
            ${{env.NORFW}}
            ${{env.NANDFW}}
            ${{env.SIZES}}

      - name: Upload firmware (rolling nightly)
        if: >-
          github.event_name != 'pull_request' &&
          (env.NORFW || env.NANDFW)
        uses: softprops/action-gh-release@v2
        with:
          tag_name: nightly
          body: |
            sha=${{ needs.preflight.outputs.head_sha }}
            short=${{ needs.preflight.outputs.short_sha }}
            built_at=${{ needs.preflight.outputs.built_at }}
          files: |
            ${{env.NORFW}}
            ${{env.NANDFW}}
            ${{env.SIZES}}

      - name: Upload firmware (latest — legacy alias)
        if: >-
          github.event_name != 'pull_request' &&
          (env.NORFW || env.NANDFW)
        uses: softprops/action-gh-release@v2
        with:
          tag_name: latest
          files: |
            ${{env.NORFW}}
            ${{env.NANDFW}}
            ${{env.SIZES}}

      - name: Send binary
        if: github.event_name != 'pull_request' && env.NORFW
        run: |
          TG_MSG="Build: ${BUILD_ID}\nCommit: ${GIT_HASH}\nBranch: ${GIT_BRANCH}\nTime: ${TIME}\n\n"
          TG_ICON="\xE2\x9C\x85 GitHub Actions"
          TG_HEADER=$(echo -e ${TG_MSG}${TG_ICON})
          TG_TOKEN=${{secrets.TELEGRAM_TOKEN_BOT_OPENIPC}}
          TG_CHANNEL=${{secrets.TELEGRAM_CHANNEL_OPENIPC_DEV}}
          HTTP=$(curl -s -o /dev/null -w %{http_code} https://api.telegram.org/bot${TG_TOKEN}/sendDocument -F chat_id=${TG_CHANNEL} -F caption="${TG_HEADER}" -F document=@${NORFW})
          echo Telegram response: ${HTTP}

  # Single umbrella status check covering the whole run, so branch protection
  # can require one context instead of a hardcoded "Build (<device>)" per
  # device -- the matrix is now dynamic, and its job names change per PR.
  # There was no gate here at all before: nothing reported a verdict on a
  # change, because nothing built one.
  ci-gate:
    name: CI Gate
    needs: [select, preflight, buildroot]
    # always() so the gate still reports when a needed job fails -- but not on
    # a run the guard above deliberately skipped, where it would read
    # select=skipped and turn a saved build into a red run.
    if: >-
      always() && (github.repository == 'OpenIPC/builder' ||
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'pull_request' && !github.event.repository.private))
    runs-on: ubuntu-latest
    steps:
      - name: Require selection + device matrix to succeed
        run: |
          echo "select=${{ needs.select.result }} preflight=${{ needs.preflight.result }} buildroot=${{ needs.buildroot.result }}"
          if [ "${{ needs.select.result }}" != "success" ]; then
            echo "::error::device selection did not succeed"; exit 1
          fi
          if [ "${{ needs.preflight.result }}" != "success" ]; then
            echo "::error::preflight did not succeed"; exit 1
          fi
          # Check the matrix against what select asked for. Blanket-accepting
          # 'skipped' would mean a selector bug that narrowed a real change
          # down to nothing reads as a green PR.
          if [ "${{ needs.select.outputs.needs-build }}" = "true" ] && \
             [ "${{ needs.preflight.outputs.should_build }}" = "true" ]; then
            if [ "${{ needs.buildroot.result }}" != "success" ]; then
              echo "::error::device matrix result=${{ needs.buildroot.result }}"; exit 1
            fi
            echo "device matrix OK (success)"
          else
            # Nothing in the change reaches a build, or the PR is still a draft.
            if [ "${{ needs.buildroot.result }}" != "skipped" ]; then
              echo "::error::device matrix ran when it should not have (${{ needs.buildroot.result }})"; exit 1
            fi
            echo "device matrix OK (no devices to build)"
          fi