From f2b5a34b91355018eb70d4d0e5c0e2851478ce92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Pe=C4=8Dovnik?= Date: Sat, 18 Jun 2022 23:56:36 +0200 Subject: [PATCH] Refactor rootfs caching (#3895) --- .github/Releases-wip.md | 2 +- .github/workflows/update-cache.yml | 323 ++++++++++++++++++++++++----- lib/configuration.sh | 2 +- 3 files changed, 278 insertions(+), 49 deletions(-) diff --git a/.github/Releases-wip.md b/.github/Releases-wip.md index 69b43d109..be25dc642 100644 --- a/.github/Releases-wip.md +++ b/.github/Releases-wip.md @@ -4,5 +4,5 @@


- Build in progress

+ Build in progress

diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml index 63e19d68d..212009b1b 100644 --- a/.github/workflows/update-cache.yml +++ b/.github/workflows/update-cache.yml @@ -1,88 +1,317 @@ -name: Update Rootfs Cache +name: Build Rootfs Cache +# +# Generates rootfs cache and uploads it to https://github.com/armbian/mirror/releases/tag/_rootfs and https://dl.armbian.com/_rootfs/ +# on: + schedule: + - cron: "30 0 1 * *" + push: + paths: + - 'config/distributions/**' + - 'config/cli/**' + - 'config/desktop/**' workflow_dispatch: - inputs: - - branch: - description: 'Build branch to use' - required: true - default: 'nightly' jobs: - merge: - uses: armbian/scripts/.github/workflows/merge-from-branch.yml@master - - with: - branch: 'nightly' - runner: small - - secrets: - GPG_KEY2: ${{ secrets.GPG_KEY2 }} - GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} - - armhf: + release-start: permissions: contents: none - needs: [ merge ] - uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master + name: Release start + runs-on: [X64] + if: ${{ github.repository_owner == 'Armbian' }} + steps: - with: - rootfsarch: 'bananapi' - runner: 'small' - branch: ${{ github.event.inputs.branch }} + - name: Checkout repository + uses: actions/checkout@v3 + with: + repository: armbian/build + path: build + clean: false - secrets: - GPG_KEY1: ${{ secrets.GPG_KEY1 }} - GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} - GPG_KEY2: ${{ secrets.GPG_KEY2 }} - GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} - SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + - name: Updating releases + uses: ncipollo/release-action@v1 + with: + repo: "mirror" + tag: "_rootfs" + name: "Build in progress" + bodyFile: "build/.github/Releases-wip.md" + allowUpdates: true + removeArtifacts: true + token: ${{ secrets.CR_PAT }} - aarch64: + x86-min: + needs: [release-start] permissions: contents: none - needs: [ merge ] - uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master with: - rootfsarch: 'lepotato' - runner: 'small' - branch: ${{ github.event.inputs.branch }} + + variant: 'minimal:uefi-x86' secrets: + PAT1: ${{ secrets.CR_PAT }} GPG_KEY1: ${{ secrets.GPG_KEY1 }} GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} GPG_KEY2: ${{ secrets.GPG_KEY2 }} GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} - amd64: + x86-srv: + needs: [release-start] permissions: contents: none - needs: [ merge ] - uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + with: - rootfsarch: 'uefi-x86' - runner: 'x64' - branch: ${{ github.event.inputs.branch }} + variant: 'server:uefi-x86' secrets: + PAT1: ${{ secrets.CR_PAT }} GPG_KEY1: ${{ secrets.GPG_KEY1 }} GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} GPG_KEY2: ${{ secrets.GPG_KEY2 }} GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + x86-x: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + + variant: 'desktop:uefi-x86' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + armhf-min: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + + variant: 'minimal:tinkerboard' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + + armhf-srv: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + + variant: 'server:tinkerboard' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + armhf-x: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + variant: 'desktop:tinkerboard' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + arm64-min: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + + variant: 'minimal:uefi-arm64' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + arm64-srv: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + + variant: 'server:uefi-arm64' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + + arm64-x: + needs: [release-start] + permissions: + contents: none + uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master + + with: + variant: 'desktop:uefi-arm64' + + secrets: + PAT1: ${{ secrets.CR_PAT }} + GPG_KEY1: ${{ secrets.GPG_KEY1 }} + GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }} + GPG_KEY2: ${{ secrets.GPG_KEY2 }} + GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }} + SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} + SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }} + KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }} jobsend: permissions: contents: none name: finish - needs: [armhf,aarch64,amd64] - runs-on: [ubuntu-latest] + needs: [x86-min,x86-srv,x86-x,armhf-min,armhf-srv,armhf-x,arm64-min,arm64-srv,arm64-x] + runs-on: [fast] if: ${{ github.repository_owner == 'Armbian' }} steps: - - run: | - echo "End" + + - name: Runner cleanup + uses: igorpecovnik/freespace@main + + - name: Checkout repository + uses: actions/checkout@v3 + with: + repository: armbian/build + path: build + clean: false + + - name: Checkout tracker lists + uses: actions/checkout@v3 + with: + fetch-depth: '1' + repository: ngosang/trackerslist + path: trackerslist + clean: false + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@v3 + with: + gpg-private-key: ${{ secrets.GPG_KEY1 }} + passphrase: ${{ secrets.GPG_PASSPHRASE1 }} + workdir: build + git-user-signingkey: true + + - name: Install SSH key for storage + uses: shimataro/ssh-key-action@v2 + with: + + key: ${{ secrets.KEY_TORRENTS }} + known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }} + if_key_exists: replace + + - name: Mount upload folders + run: | + + sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true + sudo apt-get -y -qq install sshfs + sudo mkdir -p build/cache/rootfs.upload || true + if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then + sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload + else + sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true + fi + # remove true in sshfs when all runners are on jammy + sudo df + + - name: Signing + run: | + + sudo apt-get -y -qq install parallel buildtorrent + + ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | head -1)ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1) + TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-) + WEBSEEDS="--webseeds=https://github.com/armbian/mirror/releases/download/_rootfs/$FILE,https://imola.armbian.com/dl/_rootfs/_rootfs/$FILE,https://stpete-mirror.armbian.com/dl/_rootfs/$FILE" + cd build/cache/rootfs.upload + FILES=$(ls -1 *.lz4) + for FILE in ${FILES[@]} + do + if [[ ! -f $FILE.asc ]]; then + echo "$FILE" + fi + done | sudo --preserve-env parallel --jobs 18 ' + echo "Signing {} "; echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback {}; + echo "Generating {}.torrent "; buildtorrent -q -s -m '$WEBSEEDS' --announce="'$ANNOUNCE'" --announcelist="'$TRACKERS'" {} -c "Armbian rootfs cache" {}.torrent >/dev/null + ' + + - name: Upload + uses: ncipollo/release-action@v1 + if: ${{ github.repository_owner == 'Armbian' }} + with: + repo: "mirror" + artifacts: "build/cache/rootfs/*.torrent,build/cache/rootfs/*.asc" + tag: "_rootfs" + bodyFile: "build/.github/Releases.md" + name: "Rootfs cache" + allowUpdates: true + token: ${{ secrets.PAT }} + + - name: Unmount folders + + run: | + + sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true + sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true diff --git a/lib/configuration.sh b/lib/configuration.sh index 57531b892..a400244c3 100644 --- a/lib/configuration.sh +++ b/lib/configuration.sh @@ -29,7 +29,7 @@ HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2) [[ -z $EXIT_PATCHING_ERROR ]] && EXIT_PATCHING_ERROR="" # exit patching if failed [[ -z $HOST ]] && HOST="$BOARD" # set hostname to the board cd "${SRC}" || exit -[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=19 +[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=20 [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7 BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null | grep origin) 2>/dev/null) BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)