mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
Refactor rootfs caching (#3895)
This commit is contained in:
Igor Pečovnik
2
.github/Releases-wip.md
vendored
2
.github/Releases-wip.md
vendored
@@ -4,5 +4,5 @@
|
||||
<br>
|
||||
<img src=https://raw.githubusercontent.com/armbian/.github/master/profile/progress.gif width=50>
|
||||
<br><br>
|
||||
<a href="https://github.com/armbian/build/actions/workflows/build-images.yml">Build in progress</a></p>
|
||||
<a href="https://github.com/armbian/build/actions">Build in progress</a></p>
|
||||
</p>
|
||||
|
||||
309
.github/workflows/update-cache.yml
vendored
309
.github/workflows/update-cache.yml
vendored
@@ -1,88 +1,317 @@
|
||||
name: Update Rootfs Cache
|
||||
name: Build Rootfs Cache
|
||||
#
|
||||
# Generates rootfs cache and uploads it to https://github.com/armbian/mirror/releases/tag/_rootfs and https://dl.armbian.com/_rootfs/
|
||||
#
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 0 1 * *"
|
||||
push:
|
||||
paths:
|
||||
- 'config/distributions/**'
|
||||
- 'config/cli/**'
|
||||
- 'config/desktop/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
|
||||
branch:
|
||||
description: 'Build branch to use'
|
||||
required: true
|
||||
default: 'nightly'
|
||||
|
||||
jobs:
|
||||
|
||||
merge:
|
||||
uses: armbian/scripts/.github/workflows/merge-from-branch.yml@master
|
||||
|
||||
with:
|
||||
branch: 'nightly'
|
||||
runner: small
|
||||
|
||||
secrets:
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
|
||||
armhf:
|
||||
release-start:
|
||||
permissions:
|
||||
contents: none
|
||||
needs: [ merge ]
|
||||
uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
|
||||
name: Release start
|
||||
runs-on: [X64]
|
||||
if: ${{ github.repository_owner == 'Armbian' }}
|
||||
steps:
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: armbian/build
|
||||
path: build
|
||||
clean: false
|
||||
|
||||
- name: Updating releases
|
||||
uses: ncipollo/release-action@v1
|
||||
with:
|
||||
repo: "mirror"
|
||||
tag: "_rootfs"
|
||||
name: "Build in progress"
|
||||
bodyFile: "build/.github/Releases-wip.md"
|
||||
allowUpdates: true
|
||||
removeArtifacts: true
|
||||
token: ${{ secrets.CR_PAT }}
|
||||
|
||||
x86-min:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
rootfsarch: 'bananapi'
|
||||
runner: 'small'
|
||||
branch: ${{ github.event.inputs.branch }}
|
||||
|
||||
variant: 'minimal:uefi-x86'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
aarch64:
|
||||
x86-srv:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
needs: [ merge ]
|
||||
uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
rootfsarch: 'lepotato'
|
||||
runner: 'small'
|
||||
branch: ${{ github.event.inputs.branch }}
|
||||
|
||||
variant: 'server:uefi-x86'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
amd64:
|
||||
x86-x:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
needs: [ merge ]
|
||||
uses: armbian/scripts/.github/workflows/build-rootfs-cache.yml@master
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
|
||||
rootfsarch: 'uefi-x86'
|
||||
runner: 'x64'
|
||||
branch: ${{ github.event.inputs.branch }}
|
||||
variant: 'desktop:uefi-x86'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
armhf-min:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
|
||||
variant: 'minimal:tinkerboard'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
|
||||
armhf-srv:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
|
||||
variant: 'server:tinkerboard'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
armhf-x:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
variant: 'desktop:tinkerboard'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
arm64-min:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
|
||||
variant: 'minimal:uefi-arm64'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
arm64-srv:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
|
||||
variant: 'server:uefi-arm64'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
arm64-x:
|
||||
needs: [release-start]
|
||||
permissions:
|
||||
contents: none
|
||||
uses: armbian/scripts/.github/workflows/build-cache-v2.yml@master
|
||||
|
||||
with:
|
||||
variant: 'desktop:uefi-arm64'
|
||||
|
||||
secrets:
|
||||
PAT1: ${{ secrets.CR_PAT }}
|
||||
GPG_KEY1: ${{ secrets.GPG_KEY1 }}
|
||||
GPG_PASSPHRASE1: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
GPG_KEY2: ${{ secrets.GPG_KEY2 }}
|
||||
GPG_PASSPHRASE2: ${{ secrets.GPG_PASSPHRASE2 }}
|
||||
SCRIPTS_ACCESS_TOKEN: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
|
||||
SSH_KEY_TORRENTS: ${{ secrets.KEY_TORRENTS }}
|
||||
KNOWN_HOSTS_UPLOAD: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
|
||||
jobsend:
|
||||
permissions:
|
||||
contents: none
|
||||
name: finish
|
||||
needs: [armhf,aarch64,amd64]
|
||||
runs-on: [ubuntu-latest]
|
||||
needs: [x86-min,x86-srv,x86-x,armhf-min,armhf-srv,armhf-x,arm64-min,arm64-srv,arm64-x]
|
||||
runs-on: [fast]
|
||||
if: ${{ github.repository_owner == 'Armbian' }}
|
||||
steps:
|
||||
- run: |
|
||||
echo "End"
|
||||
|
||||
- name: Runner cleanup
|
||||
uses: igorpecovnik/freespace@main
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
repository: armbian/build
|
||||
path: build
|
||||
clean: false
|
||||
|
||||
- name: Checkout tracker lists
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: '1'
|
||||
repository: ngosang/trackerslist
|
||||
path: trackerslist
|
||||
clean: false
|
||||
|
||||
- name: Import GPG key
|
||||
uses: crazy-max/ghaction-import-gpg@v3
|
||||
with:
|
||||
gpg-private-key: ${{ secrets.GPG_KEY1 }}
|
||||
passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
|
||||
workdir: build
|
||||
git-user-signingkey: true
|
||||
|
||||
- name: Install SSH key for storage
|
||||
uses: shimataro/ssh-key-action@v2
|
||||
with:
|
||||
|
||||
key: ${{ secrets.KEY_TORRENTS }}
|
||||
known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
|
||||
if_key_exists: replace
|
||||
|
||||
- name: Mount upload folders
|
||||
run: |
|
||||
|
||||
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
|
||||
sudo apt-get -y -qq install sshfs
|
||||
sudo mkdir -p build/cache/rootfs.upload || true
|
||||
if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
|
||||
sudo mount nas:/tank/armbian/users.armbian.com/upload/rootfs build/cache/rootfs.upload
|
||||
else
|
||||
sudo sshfs upload@users.armbian.com:/rootfs build/cache/rootfs.upload -o IdentityFile=~/.ssh/id_rsa -o reconnect -o allow_other || true
|
||||
fi
|
||||
# remove true in sshfs when all runners are on jammy
|
||||
sudo df
|
||||
|
||||
- name: Signing
|
||||
run: |
|
||||
|
||||
sudo apt-get -y -qq install parallel buildtorrent
|
||||
|
||||
ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | head -1)ANNOUNCE=$(cat trackerslist/trackers_best_ip.txt | sed '/^$/d' | shuf -n 1)
|
||||
TRACKERS=$(cat trackerslist/trackers_all.txt | sed '/^\s*$/d' | while read line; do printf ",""${line}"; done | cut -c 2-)
|
||||
WEBSEEDS="--webseeds=https://github.com/armbian/mirror/releases/download/_rootfs/$FILE,https://imola.armbian.com/dl/_rootfs/_rootfs/$FILE,https://stpete-mirror.armbian.com/dl/_rootfs/$FILE"
|
||||
cd build/cache/rootfs.upload
|
||||
FILES=$(ls -1 *.lz4)
|
||||
for FILE in ${FILES[@]}
|
||||
do
|
||||
if [[ ! -f $FILE.asc ]]; then
|
||||
echo "$FILE"
|
||||
fi
|
||||
done | sudo --preserve-env parallel --jobs 18 '
|
||||
echo "Signing {} "; echo ${{ secrets.GPG_PASSPHRASE1 }} | gpg --quiet --armor --batch --yes --passphrase-fd 0 --detach-sign --pinentry-mode loopback {};
|
||||
echo "Generating {}.torrent "; buildtorrent -q -s -m '$WEBSEEDS' --announce="'$ANNOUNCE'" --announcelist="'$TRACKERS'" {} -c "Armbian rootfs cache" {}.torrent >/dev/null
|
||||
'
|
||||
|
||||
- name: Upload
|
||||
uses: ncipollo/release-action@v1
|
||||
if: ${{ github.repository_owner == 'Armbian' }}
|
||||
with:
|
||||
repo: "mirror"
|
||||
artifacts: "build/cache/rootfs/*.torrent,build/cache/rootfs/*.asc"
|
||||
tag: "_rootfs"
|
||||
bodyFile: "build/.github/Releases.md"
|
||||
name: "Rootfs cache"
|
||||
allowUpdates: true
|
||||
token: ${{ secrets.PAT }}
|
||||
|
||||
- name: Unmount folders
|
||||
|
||||
run: |
|
||||
|
||||
sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true
|
||||
sudo mountpoint -q build/cache/rootfs.upload && sudo fusermount -u build/cache/rootfs.upload || true
|
||||
|
||||
@@ -29,7 +29,7 @@ HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2)
|
||||
[[ -z $EXIT_PATCHING_ERROR ]] && EXIT_PATCHING_ERROR="" # exit patching if failed
|
||||
[[ -z $HOST ]] && HOST="$BOARD" # set hostname to the board
|
||||
cd "${SRC}" || exit
|
||||
[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=19
|
||||
[[ -z "${ROOTFSCACHE_VERSION}" ]] && ROOTFSCACHE_VERSION=20
|
||||
[[ -z "${CHROOT_CACHE_VERSION}" ]] && CHROOT_CACHE_VERSION=7
|
||||
BUILD_REPOSITORY_URL=$(improved_git remote get-url $(improved_git remote 2>/dev/null | grep origin) 2>/dev/null)
|
||||
BUILD_REPOSITORY_COMMIT=$(improved_git describe --match=d_e_a_d_b_e_e_f --always --dirty 2>/dev/null)
|
||||
|
||||
Reference in New Issue
Block a user