From 63994266e2d8fc2fb794235125963b1bb81a8e4a Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Sat, 23 May 2026 23:05:36 +0300 Subject: [PATCH] ci/nightly: manifest aggregator + 90-build retention sweep (#92) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-Bld-B in the mirror of OpenIPC/firmware's nightly redesign. Adds the two workflows that turn dated nightly-* releases (from PR-Bld-A) into a queryable index served via GitHub Pages. `manifest.yml` — triggers on successful `Build` workflow_run completion (or manual dispatch). Runs `.github/scripts/enrich_manifest.py` to enumerate dated releases, keep the 90 newest, parse the body (sha/short/built_at from PR-Bld-A) and asset list, then emit: - `manifest.json` — rich JSON for hosts/agents/CI. - `manifest.flat` — whitespace-delimited columns `build_id platform flash size url` plus `@channel` records, parseable by pure busybox `awk` (no jq, no jsonfilter) for on-device sysupgrade in Phase 2. Both committed to the `gh-pages` branch (already bootstrapped). `cleanup.yml` — Mondays 06:00 UTC (offset by +1h from firmware's 05:00 UTC cleanup). Deletes dated nightly releases beyond the 90 newest via `gh release delete --cleanup-tag`, then re-triggers manifest.yml. `enrich_manifest.py` differs from firmware's only in the asset parser: builder produces TWO filename forms (per `master.yml`'s COMMON awk trick that counts underscores in the matrix entry name): - **Compound** `__--nor.tgz` → platform key = the full string. This is the common case (per-device builds). - **Simple** `openipc.--.tgz` (firmware-style, when matrix name has one underscore) → platform key = `_`. The platform key gets carried into manifest.flat as a single column. sysupgrade's existing `awk '$2==p'` lookup works unchanged; Phase 3 (model-aware BUILD_PLATFORM field in os-release) will let cameras filter against the full per-model key. URLs after merge: https://openipc.github.io/builder/manifest.json https://openipc.github.io/builder/manifest.flat (`gh-pages` branch + Pages config already set up; mirrors firmware- side setup.) Concurrency-grouped `gh-pages-manifest` so manifest.yml and cleanup.yml can't race the gh-pages push. Includes the retry hardening from firmware OpenIPC/firmware#2129 (4-attempt backoff for transient GitHub API 401/5xx, fast-fail on permanent 404). First-run behaviour validated locally: empty index emits explicit placeholder. Parser unit-tested across 7 synthetic filenames covering compound, simple, and edge cases. See ~/.claude/plans/mirror-nightly-redesign-to-builder.md. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/scripts/enrich_manifest.py | 179 +++++++++++++++++++++++++++++ .github/workflows/cleanup.yml | 44 +++++++ .github/workflows/manifest.yml | 56 +++++++++ 3 files changed, 279 insertions(+) create mode 100644 .github/scripts/enrich_manifest.py create mode 100644 .github/workflows/cleanup.yml create mode 100644 .github/workflows/manifest.yml diff --git a/.github/scripts/enrich_manifest.py b/.github/scripts/enrich_manifest.py new file mode 100644 index 0000000..cdf24f4 --- /dev/null +++ b/.github/scripts/enrich_manifest.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Build manifest.json and manifest.flat for OpenIPC/builder's nightly index. + +Mirrors OpenIPC/firmware's enrich_manifest.py but accounts for builder's +**bimodal** asset filenames produced by .github/workflows/master.yml: + + - **Compound** form (one underscore per matrix-entry awk count == multi): + `__--nor.tgz` — encodes the full device + identifier. Platform key = the full `__-` + string. This is the common case (per-device builds). + + - **Simple** form (single-underscore matrix entry like `ssc338q_fpv`): + `openipc.--.tgz` — firmware-style. Platform key + = `_`. + +The flat schema is unchanged: `build_id platform flash size url`, plus +`@channel ` records. + +md5 is intentionally absent (same reasoning as firmware — every .tgz +ships a .md5sum sidecar that sysupgrade validates after download). + +Retry hardening (HTTP 401 / 5xx / network) is the same as firmware's +post-#2129 enrich_manifest, with fast-fail on permanent 404. +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path + +REPO = os.environ.get("GITHUB_REPOSITORY", "OpenIPC/builder") +RETENTION = 90 +TAG_RE = re.compile(r"^nightly-(\d{8})-([0-9a-f]{7})$") + +# Bimodal asset names — see module docstring. +COMPOUND_RE = re.compile(r"^(?P[a-z0-9]+_[a-z0-9]+_[a-z0-9_.-]+)-(?Pnor|nand)\.tgz$") +SIMPLE_RE = re.compile(r"^openipc\.(?P[^.]+)-(?Pnor|nand)-(?P\w+)\.tgz$") + +# Retry budget for transient GitHub API failures (mirrors firmware/#2129). +GH_RETRY_DELAYS = (0, 5, 15, 40) + + +def gh(*args: str) -> str: + cmd = ["gh", *args, "--repo", REPO] + last_exc = None + for delay in GH_RETRY_DELAYS: + if delay > 0: + time.sleep(delay) + try: + return subprocess.check_output(cmd, text=True, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + last_exc = e + err = e.stderr or "" + if "release not found" in err.lower() or "not found (HTTP 404)" in err: + break + sys.stderr.write( + f"gh {' '.join(args[:3])}: attempt failed " + f"(rc={e.returncode}): {err.strip()[:240]}\n" + ) + sys.stderr.write( + f"gh {' '.join(args[:3])}: giving up; " + f"final stderr:\n{last_exc.stderr}\n" + ) + raise last_exc + + +def list_dated_releases() -> list[dict]: + raw = gh("release", "list", "--limit", "200", + "--json", "tagName,createdAt,isPrerelease") + rels = json.loads(raw) + dated = [r for r in rels if TAG_RE.match(r["tagName"])] + dated.sort(key=lambda r: r["createdAt"], reverse=True) + return dated[:RETENTION] + + +def fetch_release(tag: str) -> dict: + raw = gh("release", "view", tag, + "--json", "tagName,createdAt,body,assets") + return json.loads(raw) + + +def parse_body(body: str | None) -> tuple[str, str, str]: + sha = short = built_at = "" + for line in (body or "").splitlines(): + if line.startswith("sha="): + sha = line[4:].strip() + elif line.startswith("short="): + short = line[6:].strip() + elif line.startswith("built_at="): + built_at = line[9:].strip() + return sha, short, built_at + + +def parse_asset(name: str) -> tuple[str, str] | None: + """Returns (platform_key, flash) or None if filename doesn't match.""" + m = COMPOUND_RE.match(name) + if m: + return m.group("platform"), m.group("flash") + m = SIMPLE_RE.match(name) + if m: + return f"{m.group('soc')}_{m.group('variant')}", m.group("flash") + return None + + +def main() -> None: + out_dir = Path(sys.argv[1] if len(sys.argv) > 1 else ".") + out_dir.mkdir(parents=True, exist_ok=True) + + now = dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + dated = list_dated_releases() + + if not dated: + manifest = {"schema": 1, "generated_at": now, + "channels": {}, "builds": []} + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + (out_dir / "manifest.flat").write_text( + f"# generated_at={now}\n" + "# No nightly-YYYYMMDD- releases yet — " + "the first scheduled build will populate this index.\n" + ) + print("manifest: 0 builds (empty index)") + return + + builds = [] + for rel in dated: + info = fetch_release(rel["tagName"]) + sha, short, built_at = parse_body(info.get("body") or "") + platforms: dict[str, dict[str, dict]] = {} + for a in info.get("assets") or []: + parsed = parse_asset(a["name"]) + if not parsed: + continue + platform, flash = parsed + platforms.setdefault(platform, {})[flash] = { + "url": a["url"], + "size": a["size"], + } + builds.append({ + "id": info["tagName"], + "sha": sha, + "short": short, + "built_at": built_at or info["createdAt"], + "release_url": f"https://github.com/{REPO}/releases/tag/{info['tagName']}", + "platforms": platforms, + }) + + newest = builds[0]["id"] + manifest = { + "schema": 1, + "generated_at": now, + "channels": {"nightly": newest, "latest": newest}, + "builds": builds, + } + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + + lines = [ + "# OpenIPC builder build index", + f"# generated_at={now}", + "# columns: build_id platform flash size url", + ] + for b in builds: + for platform, flashes in sorted(b["platforms"].items()): + for flash, info in sorted(flashes.items()): + lines.append(f"{b['id']} {platform} {flash} {info['size']} {info['url']}") + lines.append("# channels") + for ch, target in manifest["channels"].items(): + lines.append(f"@channel {ch} {target}") + (out_dir / "manifest.flat").write_text("\n".join(lines) + "\n") + + print(f"manifest: {len(builds)} builds, newest={newest}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000..f37fbe4 --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,44 @@ +name: cleanup +on: + schedule: + - cron: '0 6 * * 1' # Mondays 06:00 UTC (offset by +1h from firmware's cleanup) + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: gh-pages-manifest + cancel-in-progress: false + +jobs: + prune: + name: Prune old nightly releases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Delete releases beyond the 90 newest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + to_delete=$(gh release list --limit 200 --json tagName,createdAt \ + | jq -r '[ .[] | select(.tagName | test("^nightly-[0-9]{8}-[0-9a-f]{7}$")) ] + | sort_by(.createdAt) | reverse | .[90:] | .[].tagName') + + if [ -z "$to_delete" ]; then + echo "Nothing to delete; <=90 dated nightlies present." + exit 0 + fi + + echo "$to_delete" | while read -r tag; do + [ -z "$tag" ] && continue + echo "Deleting $tag" + gh release delete "$tag" --cleanup-tag --yes + done + + - name: Refresh manifest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run manifest.yml diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml new file mode 100644 index 0000000..58e3e57 --- /dev/null +++ b/.github/workflows/manifest.yml @@ -0,0 +1,56 @@ +name: manifest +on: + workflow_run: + workflows: [Build] + types: [completed] + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: gh-pages-manifest + cancel-in-progress: false + +jobs: + generate: + name: Generate manifest + # Publish on success AND on partial-failure: one flaky board (e.g. a + # transient toolchain 502) should not deny manifest updates for the + # platforms that did upload artifacts. enrich_manifest.py reads + # whatever assets actually exist on the release. + if: >- + github.event_name == 'workflow_dispatch' || + contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-latest + steps: + - name: Checkout master (for the script) + uses: actions/checkout@v4 + with: + path: master + + - name: Checkout gh-pages (for output) + uses: actions/checkout@v4 + with: + ref: gh-pages + path: pages + + - name: Generate manifest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: python3 master/.github/scripts/enrich_manifest.py pages + + - name: Publish to gh-pages + working-directory: pages + run: | + git config user.email "actions@github.com" + git config user.name "github-actions[bot]" + git add manifest.json manifest.flat + if git diff --cached --quiet; then + echo "No manifest changes; nothing to commit." + else + newest=$(jq -r '.channels.nightly // "empty"' manifest.json) + git commit -m "manifest: $(date -u +%FT%TZ) — ${newest}" + git push + fi