name: Image, disk image and live ISO

# Builds the quine-linux image on our Aurora build (published by
# quine-global-labs/aurora's workflow to this forge's registry), pushes it to
# forge.in.quineglobal.com/quine-global-labs/quine-linux (`latest`,
# `<date>`, the commit; installed systems pull it as
# registry.quineglobal.com/quine-linux, a pull-only name for it), then
# builds a disk image (qcow2) and the live ISO
# (which is also the installer, as Aurora's) from it and uploads them to MinIO
# as <bucket>/disk/latest/disk.qcow2 and <bucket>/live/latest/live.iso
# (aarch64: disk-aarch64.qcow2, live-aarch64.iso), replacing the last build's.
#
# Once per architecture, each natively on its own runner (host mode, root
# podman; petersweb-infra, nixos/forgejo-runner.nix): x86_64 on
# runner-x86-linux, from aurora:stable, and aarch64 on runner-arm64-linux (a
# Tart VM on the Mac mini), from aurora:stable-arm64. bootc-image-builder
# needs a host of the image's architecture with root podman and ~30 GB of
# free disk. aarch64's image is quine-linux:latest-arm64 (its other tags end
# in -arm64 too), and installs follow that; its files end in -aarch64.
#
# A tag `v*` also publishes a release: each architecture's files go to
# MinIO's public <bucket>/releases/<tag>/, and then the release job writes
# one SHA256SUMS for all of them and makes a Forgejo release on this repo
# that links to them
# (Forgejo's external-URL release files: the images are too big for GitHub's
# 2 GiB limit, and this keeps one copy). The website lists these releases.
#
# Upstream images (Fedora's, bootc-image-builder) are vendored into
# forge.in.quineglobal.com/quine-global-labs/mirror/ first, and pulled
# from there (.forgejo/vendor-images.sh).
#
# Secrets: REGISTRY_TOKEN (a Forgejo token with package write access),
# MINIO_SECRET_KEY. MINIO_ACCESS_KEY is a variable or a secret (it's just the
# MinIO user's name, `quine-linux`).
# Variables (optional): REGISTRY_USER (the token's owner; default: the
# actor), MINIO_ENDPOINT (default https://s3.quineglobal.com), MINIO_BUCKET
# (default quine-linux).

# Also daily, two hours after quine-global-labs/aurora's daily build, so the
# image and ISO pick up its result. Forgejo reads the schedule from pushes to
# main, but not from `[skip ci]` ones: after changing it, push without.
on:
  push:
    branches: [main]
    tags: ['v*']
  schedule:
    - cron: '0 7 * * *'
  workflow_dispatch:

concurrency:
  group: iso-${{ github.ref }}
  cancel-in-progress: true

env:
  REGISTRY: forge.in.quineglobal.com
  IMAGE: forge.in.quineglobal.com/quine-global-labs/quine-linux
  MINIO_ENDPOINT: ${{ vars.MINIO_ENDPOINT || 'https://s3.quineglobal.com' }}
  MINIO_BUCKET: ${{ vars.MINIO_BUCKET || 'quine-linux' }}
  # rclone's remote `store:`, all from the environment (no rclone.conf); each
  # step that uses it adds RCLONE_CONFIG_STORE_ACCESS_KEY_ID/_SECRET_ACCESS_KEY.
  # rclone, not mc: MinIO abandoned mc, and nixpkgs marks it insecure.
  RCLONE_CONFIG_STORE_TYPE: s3
  RCLONE_CONFIG_STORE_PROVIDER: Minio
  RCLONE_CONFIG_STORE_ENDPOINT: ${{ vars.MINIO_ENDPOINT || 'https://s3.quineglobal.com' }}

jobs:
  iso:
    strategy:
      # One architecture failing doesn't stop the other.
      fail-fast: false
      matrix:
        include:
          - arch: amd64
            runner: runner-x86-linux
            platform: linux/amd64
            file-arch: x86_64
            # Appended to the image's tags, and to the latest/ files' names.
            tag-suffix: ''
            latest-suffix: ''
          - arch: arm64
            runner: runner-arm64-linux
            platform: linux/arm64
            file-arch: aarch64
            tag-suffix: -arm64
            latest-suffix: -aarch64
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 300
    env:
      BASE_IMAGE: forge.in.quineglobal.com/quine-global-labs/aurora:stable${{ matrix.tag-suffix }}
      # What installs of this architecture follow for updates (Taskfile.yml).
      UPDATE_IMAGE: registry.quineglobal.com/quine-linux:latest${{ matrix.tag-suffix }}
      # The tasks' architecture settings (Taskfile.yml).
      ARCH_VARS: PLATFORM=${{ matrix.platform }} TARGET_ARCH=${{ matrix.arch }}
    steps:
      # No submodules: stage 0 (Aurora) comes from the registry now. Our pull
      # mirror of code.forgejo.org/actions/checkout.
      - uses: https://forge.in.quineglobal.com/quine-global-labs/mirror-actions-checkout@v4

      - name: Install task, rclone, just, yq and cosign
        # Only what the runner doesn't already have: both runners have all
        # five from nixpkgs (petersweb-infra, nixos/forgejo-runner.nix).
        run: |
          bin="$HOME/.local/bin"
          mkdir -p "$bin"
          have() { command -v "$1" >/dev/null; }
          have task || curl -fsSL https://taskfile.dev/install.sh | sh -s -- -d -b "$bin"
          # --force: on a host-mode runner $HOME outlives the job.
          have just || curl -fsSL https://just.systems/install.sh | bash -s -- --force --to "$bin"
          have yq || { curl -fsSL -o "$bin/yq" https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && chmod +x "$bin/yq"; }
          have cosign || { curl -fsSL -o "$bin/cosign" https://github.com/sigstore/cosign/releases/latest/download/cosign-linux-amd64 && chmod +x "$bin/cosign"; }
          have rclone || { curl -fsSL -o /tmp/rclone.zip https://downloads.rclone.org/rclone-current-linux-amd64.zip && unzip -joq /tmp/rclone.zip '*/rclone' -d "$bin" && rm /tmp/rclone.zip; }
          echo "$bin" >> "$GITHUB_PATH"
          for t in task just yq cosign rclone; do printf '%s: ' "$t"; command -v "$t"; done

      - name: Remove what failed builds left behind
        # A build that fails or is cancelled can leave its build containers
        # (`podman ps --external`), and they keep their images, ~11 GB each,
        # from every prune (run 48 ran out of disk). One job per runner
        # (petersweb-infra, nixos/forgejo-runner.nix), so none is ours. And a
        # cancelled run skips the cleanup at the end: its images, 25 GB on
        # arm64, left too little of the 50 GB disk for bootc-image-builder
        # (run 50, attempt 2). Every job ends with prune -af anyway.
        run: |
          ids=$(podman ps -aq --external)
          [ -z "$ids" ] || { echo "removing leftover build containers: $ids"; podman rm -f $ids || true; }
          podman image prune -af >/dev/null || true
          df -h /var/lib/containers/storage | tail -1

      - name: Log in to the registry
        env:
          REGISTRY_USER: ${{ vars.REGISTRY_USER || github.actor }}
          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
        run: printf %s "$REGISTRY_TOKEN" | podman login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin

      - name: Vendor upstream images
        # Into the forge's mirror/ (.forgejo/vendor-images.sh), which the
        # pulls below then read: bootc-image-builder (Taskfile.yml) and what
        # Titanoboa's ISO step runs in (Taskfile.yml, FEDORA_IMAGE).
        run: |
          .forgejo/vendor-images.sh \
            quay.io/fedora/fedora:latest \
            ghcr.io/osbuild/bootc-image-builder:latest

      - name: Pull base images
        # Before the builds, with retries: quay.io's CDN cuts off multi-GB
        # layers now and then ("unexpected EOF"), and a pull inside
        # `buildah build` gives up after one reconnect. Already-pulled
        # images cost only a manifest check. The upstream ones come from the
        # forge's mirror/ when it has them (runner-x86-linux's
        # registries.conf).
        run: |
          for img in \
            "$BASE_IMAGE" \
            quay.io/fedora/fedora:latest \
            ghcr.io/osbuild/bootc-image-builder:latest; do
            start=$SECONDS
            for try in 1 2 3 4 5; do
              echo "== pulling $img (attempt $try/5)"
              podman pull --retry 5 --retry-delay 20s "$img" && break
              [ "$try" != 5 ] || { echo "== giving up on $img after 5 attempts"; exit 1; }
              echo "== pull of $img failed, retrying in 30s"; sleep 30
            done
            podman image inspect --format \
              "== $img: {{.Digest}} {{.Os}}/{{.Architecture}}, {{.Size}} bytes, $((SECONDS - start))s" \
              "$img"
          done

      - name: Build image
        run: task build-image CONTAINER_RUNTIME=podman BASE_IMAGE="$BASE_IMAGE" $ARCH_VARS

      - name: Push image
        env:
          SUFFIX: ${{ matrix.tag-suffix }}
        run: |
          tags=(latest "$(date -u +%Y%m%d)" "${GITHUB_SHA::12}")
          [ "$GITHUB_REF_TYPE" != tag ] || tags+=("$GITHUB_REF_NAME")
          for t in "${tags[@]}"; do
            podman tag localhost/quine-linux "$IMAGE:$t$SUFFIX"
            podman push --retry 5 "$IMAGE:$t$SUFFIX"
          done

      - name: Build disk image
        run: task build-disk-image CONTAINER_RUNTIME=podman BASE_IMAGE="$BASE_IMAGE" UPDATE_IMAGE="$UPDATE_IMAGE" $ARCH_VARS

      - name: Build live ISO
        # Installs UPDATE_IMAGE (quine-linux:latest, or latest-arm64), which
        # "Push image" just updated. Pulled into the ISO straight from the
        # forge, under this commit's tag (PAYLOAD_PULL): through the
        # registry.quineglobal.com proxy (Hetzner and back) it ran at ~1 MB/s
        # against ~100 MB/s direct, and took most of this step.
        env:
          SUFFIX: ${{ matrix.tag-suffix }}
        run: >-
          task build-live-iso CONTAINER_RUNTIME=podman BASE_IMAGE="$BASE_IMAGE"
          UPDATE_IMAGE="$UPDATE_IMAGE" PAYLOAD_PULL="$IMAGE:${GITHUB_SHA::12}$SUFFIX" $ARCH_VARS

      - name: Upload to MinIO
        env:
          RCLONE_CONFIG_STORE_ACCESS_KEY_ID: ${{ vars.MINIO_ACCESS_KEY || secrets.MINIO_ACCESS_KEY }}
          RCLONE_CONFIG_STORE_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_KEY }}
          SUFFIX: ${{ matrix.latest-suffix }}
        run: |
          rclone mkdir "store:${MINIO_BUCKET}"
          # Latest only: each file is 5-8 GB, and the forge VM's disk is
          # shared. Tagged releases keep their own copies (releases/<tag>/).
          ls -lh output/qcow2/disk.qcow2
          rclone copyto output/qcow2/disk.qcow2 "store:${MINIO_BUCKET}/disk/latest/disk$SUFFIX.qcow2"
          ls -lh output/live/Quine-Linux-Live.iso
          rclone copyto output/live/Quine-Linux-Live.iso "store:${MINIO_BUCKET}/live/latest/live$SUFFIX.iso"

      - name: Upload the release's files
        # The release job below lists them in the release, with one
        # SHA256SUMS for both architectures from the SHA256SUMS.<arch> parts.
        if: github.ref_type == 'tag'
        env:
          RCLONE_CONFIG_STORE_ACCESS_KEY_ID: ${{ vars.MINIO_ACCESS_KEY || secrets.MINIO_ACCESS_KEY }}
          RCLONE_CONFIG_STORE_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_KEY }}
          ARCH: ${{ matrix.file-arch }}
        run: |
          tag=$GITHUB_REF_NAME
          mkdir -p release
          ln -f output/qcow2/disk.qcow2 "release/quine-linux-$tag-$ARCH.qcow2"
          ln -f output/live/Quine-Linux-Live.iso "release/quine-linux-$tag-live-$ARCH.iso"
          (cd release && sha256sum -- * > "SHA256SUMS.$ARCH")
          cat "release/SHA256SUMS.$ARCH"

          rclone copy release/ "store:${MINIO_BUCKET}/releases/$tag/"

      - name: Clean up
        if: always()
        run: |
          rm -rf output release
          podman logout "$REGISTRY" || true
          # Each build leaves ~100 GB of images and stages; the runner's daily
          # prune keeps a day's worth, and a second build that day filled the
          # disk. The next build pulls its bases from the forge again.
          ids=$(podman ps -aq --external)
          [ -z "$ids" ] || podman rm -f $ids || true
          podman image prune -af >/dev/null || true

  release:
    # After both architectures, even if one failed: the release gets
    # whatever was built (SHA256SUMS says which).
    needs: iso
    if: always() && github.ref_type == 'tag'
    runs-on: runner-x86-linux
    steps:
      - name: Publish the release
        env:
          RCLONE_CONFIG_STORE_ACCESS_KEY_ID: ${{ vars.MINIO_ACCESS_KEY || secrets.MINIO_ACCESS_KEY }}
          RCLONE_CONFIG_STORE_SECRET_ACCESS_KEY: ${{ secrets.MINIO_SECRET_KEY }}
          TOKEN: ${{ github.token }}
        run: |
          tag=$GITHUB_REF_NAME
          dir="store:${MINIO_BUCKET}/releases/$tag"
          sums=$(mktemp)
          for part in $(rclone lsf --files-only --include 'SHA256SUMS.*' "$dir/" | sort); do
            rclone cat "$dir/$part" >>"$sums"
          done
          [ -s "$sums" ] || { echo "::error::no architecture uploaded its files"; exit 1; }
          cat "$sums"
          rclone copyto "$sums" "$dir/SHA256SUMS"
          files=(SHA256SUMS)
          mapfile -t -O 1 files < <(awk '{print $2}' "$sums")

          api="$GITHUB_SERVER_URL/api/v1/repos/$GITHUB_REPOSITORY/releases"
          auth=(-H "Authorization: token $TOKEN")
          # v1.2.3-alpha.1 and the like: a pre-release.
          pre=false; [[ $tag != *-* ]] || pre=true
          id=$(jq -n --arg tag "$tag" --arg image "$IMAGE:$tag" --argjson pre "$pre" '{tag_name: $tag, name: $tag,
                 prerelease: $pre, body: "Container images: `\($image)` (x86_64), `\($image)-arm64` (aarch64).\nFiles are checked by SHA256SUMS."}' |
               curl -fsS "${auth[@]}" -H 'Content-Type: application/json' -d @- "$api" | jq -er .id)
          for name in "${files[@]}"; do
            curl -fsS "${auth[@]}" -F "external_url=${MINIO_ENDPOINT}/${MINIO_BUCKET}/releases/$tag/$name" \
              "$api/$id/assets?name=$name" >/dev/null
            echo "linked $name"
          done
          echo "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$tag"