Refactor rootfs caching (#3895)

.github/Releases-wip.md

@@ -4,5 +4,5 @@
   <br>
   <img src=https://raw.githubusercontent.com/armbian/.github/master/profile/progress.gif width=50>
   <br><br>
-  <a href="https://github.com/armbian/build/actions/workflows/build-images.yml">Build in progress</a></p>
+  <a href="https://github.com/armbian/build/actions">Build in progress</a></p>
 </p>

.github/workflows/update-cache.yml

@@ -1,88 +1,317 @@
-name: Update Rootfs Cache
+name: Build Rootfs Cache
+#
+# Generates rootfs cache and uploads it to https://github.com/armbian/mirror/releases/tag/_rootfs and https://dl.armbian.com/_rootfs/
+#
 
 on:
+  schedule:
+    - cron: "30 0 1 * *"
+  push:
+      paths:
+      - 'config/distributions/**'
+      - 'config/cli/**'
+      - 'config/desktop/**'
   workflow_dispatch:
-    inputs:
-
-      branch:
-        description: 'Build branch to use'
-        required: true
-        default: 'nightly'
 
 jobs:
 
-  merge:
+  release-start:
-    uses: armbian/scripts/.github/workflows/merge-from-branch.yml@master
-
-    with:
-      branch: 'nightly'
-      runner: small
-
-    secrets:  
-      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
-      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
-
-  armhf:
     permissions:
       contents: none
-    needs: [ merge ]
+    name: Release start
-    uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
+    runs-on: [X64]
+    if: ${{ github.repository_owner == 'Armbian' }}
+    steps:
 
-    with:
+      - name: Checkout repository
-      rootfsarch: 'bananapi'
+        uses: actions/checkout@v3
-      runner: 'small'
+        with:
-      branch: ${{ github.event.inputs.branch }}
+          repository: armbian/build
+          path: build
+          clean: false
 
-    secrets:  
+      - name: Updating releases
-      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+        uses: ncipollo/release-action@v1
-      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+        with:
-      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+          repo: "mirror"
-      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+          tag: "_rootfs"
-      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+          name: "Build in progress"
+          bodyFile: "build/.github/Releases-wip.md"
+          allowUpdates: true
+          removeArtifacts: true
+          token: ${{ secrets.CR_PAT }}
 
-  aarch64:
+  x86-min:
+    needs: [release-start]
     permissions:
       contents: none
-    needs: [ merge ]
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
-    uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
 
     with:
-      rootfsarch: 'lepotato'
+
-      runner: 'small'
+      variant: 'minimal:uefi-x86'
-      branch: ${{ github.event.inputs.branch }}
 
     secrets:
+      PAT1: ${{ secrets.CR_PAT }}
       GPG_KEY1: ${{ secrets.GPG_KEY1 }}
       GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
       GPG_KEY2: ${{ secrets.GPG_KEY2 }}
       GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
       SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
 
-  amd64:
+  x86-srv:
+    needs: [release-start]
     permissions:
       contents: none
-    needs: [ merge ]
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
-    uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
+
     with:
 
-      rootfsarch: 'uefi-x86'
+      variant: 'server:uefi-x86'
-      runner: 'x64'
-      branch: ${{ github.event.inputs.branch }}
 
     secrets:
+      PAT1: ${{ secrets.CR_PAT }}
       GPG_KEY1: ${{ secrets.GPG_KEY1 }}
       GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
       GPG_KEY2: ${{ secrets.GPG_KEY2 }}
       GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
       SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  x86-x:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+
+      variant: 'desktop:uefi-x86'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  armhf-min:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+
+      variant: 'minimal:tinkerboard'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+
+  armhf-srv:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+
+      variant: 'server:tinkerboard'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  armhf-x:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+      variant: 'desktop:tinkerboard'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  arm64-min:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+
+      variant: 'minimal:uefi-arm64'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  arm64-srv:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+
+      variant: 'server:uefi-arm64'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+
+  arm64-x:
+    needs: [release-start]
+    permissions:
+      contents: none
+    uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
+
+    with:
+      variant: 'desktop:uefi-arm64'
+
+    secrets:
+      PAT1: ${{ secrets.CR_PAT }}
+      GPG_KEY1: ${{ secrets.GPG_KEY1 }}
+      GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY2 }}
+      GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
+      SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
+      SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
+      KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
 
   jobsend:
     permissions:
       contents: none
     name: finish
-    needs: [armhf,aarch64,amd64]
+    needs: [x86-min,x86-srv,x86-x,armhf-min,armhf-srv,armhf-x,arm64-min,arm64-srv,arm64-x]
-    runs-on: [ubuntu-latest]
+    runs-on: [fast]
     if: ${{ github.repository_owner == 'Armbian' }}
     steps:
-      - run: |
+
-          echo "End"
+      - name: Runner cleanup
+        uses: igorpecovnik/freespace@main
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          repository: armbian/build
+          path: build
+          clean: false
+
+      - name: Checkout tracker lists
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: '1'
+          repository: ngosang/trackerslist
+          path: trackerslist
+          clean: false
+
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v3
+        with:
+          gpg-private-key: ${{ secrets.GPG_KEY1 }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
+          workdir: build
+          git-user-signingkey: true
+
+      - name: Install SSH key for storage
+        uses: shimataro/ssh-key-action@v2
+        with:
+
+          key: ${{ secrets.KEY_TORRENTS }}
+          known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
+          if_key_exists: replace
+
+      - name: Mount upload folders
+        run: |
+
+          sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
+          sudo apt-get -y -qq install sshfs
+          sudo mkdir -p build/cache/rootfs.upload || true
+          if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
+              sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload
+          else
+              sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
+          fi
+          # remove true in sshfs when all runners are on jammy
+          sudo df
+
+      - name: Signing
+        run: |
+
+          sudo apt-get -y -qq install parallel buildtorrent
+
+          ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | head -1)ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1)
+          TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)
+          WEBSEEDS="--webseeds=https://github.com/armbian/mirror/releases/download/_rootfs/$FILE,https://imola.armbian.com/dl/_rootfs/_rootfs/$FILE,https://stpete-mirror.armbian.com/dl/_rootfs/$FILE"
+          cd build/cache/rootfs.upload
+          FILES=$(ls -1 *.lz4)
+          for FILE in ${FILES[@]}
+          do
+          if [[ ! -f $FILE.asc ]]; then
+              echo "$FILE"
+          fi
+          done | sudo --preserve-env parallel --jobs 18 '
+              echo "Signing {} "; echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback {};
+              echo "Generating {}.torrent "; buildtorrent -q -s -m '$WEBSEEDS' --announce="'$ANNOUNCE'" --announcelist="'$TRACKERS'" {} -c "Armbian rootfs cache" {}.torrent >/dev/null
+          '
+
+      - name: Upload
+        uses: ncipollo/release-action@v1
+        if: ${{ github.repository_owner == 'Armbian' }}
+        with:
+          repo: "mirror"
+          artifacts: "build/cache/rootfs/*.torrent,build/cache/rootfs/*.asc"
+          tag: "_rootfs"
+          bodyFile: "build/.github/Releases.md"
+          name: "Rootfs cache"
+          allowUpdates: true
+          token: ${{ secrets.PAT }}
+
+      - name: Unmount folders
+
+        run: |
+
+          sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
+          sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true

lib/configuration.sh

@@ -29,7 +29,7 @@ HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2)
 [[ -z $EXIT_PATCHING_ERROR ]] && EXIT_PATCHING_ERROR="" # exit patching if failed
 [[ -z $HOST ]] && HOST="$BOARD" # set hostname to the board
 cd "${SRC}" || exit
-[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=19
+[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=20
 [[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7
 BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null | grep origin) 2>/dev/null)
 BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)