From eb60188ad1cc7ebeb4eff4d5a40760f9bd0039d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Pe=C4=8Dovnik?= Date: Tue, 9 Nov 2021 18:31:48 +0100 Subject: [PATCH] Update Github actions scripts (#3209) --- .github/labeler.yml | 10 +- .github/workflows/README.md | 2 +- .github/workflows/build-beta-images.yml | 64 ++++++++- .github/workflows/build-beta-kernel.yml | 22 ++- .github/workflows/build-cache.yml | 70 ++++++--- .github/workflows/maintain.yml | 179 +----------------------- 6 files changed, 134 insertions(+), 213 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 49d1ece20..1144c96b8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -24,6 +24,12 @@ - config/kernel/* - config/sources/* +"c:bsp": + - config/sources/**/* + - config/boards/* + - config/bootscripts/* + - config/bootenv/* + "c:build script": - lib/* @@ -33,5 +39,5 @@ "c:cli": - config/cli/* -"c:ci": - - .github/* +"c:actions": + - .github/workflows/* diff --git a/.github/workflows/README.md b/.github/workflows/README.md index fddc498ae..6c885c081 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -5,7 +5,7 @@ Common tags: - self-hosted - Linux - X64 -- ARM64 +- ARM64 (4Gb memory with ZRAM_PERCENTAGE=50) - public (isolated runners for merge reqeusts) - local (local network) - cache (mounted cache) diff --git a/.github/workflows/build-beta-images.yml b/.github/workflows/build-beta-images.yml index 11e22ea58..578f31f46 100644 --- a/.github/workflows/build-beta-images.yml +++ b/.github/workflows/build-beta-images.yml @@ -13,8 +13,48 @@ on: # - completed jobs: + Merge: + + name: "Merge master into nightly" + runs-on: [self-hosted, Linux] + if: ${{ github.repository_owner == 'Armbian' }} + + steps: + + - name: Checkout Armbian build script + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: armbian/build + path: build + ref: nightly + clean: false + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@v3 + with: + gpg-private-key: ${{ secrets.GPG_KEY2 }} + passphrase: ${{ secrets.GPG_PASSPHRASE2 }} + workdir: build + git-user-signingkey: true + git-commit-gpgsign: true + + - name: Merge master into nightly + + run: | + cd build + git config --global user.email "info@armbian.com" + git config --global user.name "Armbianworker" + git checkout master + git fetch + git merge origin/master + git checkout nightly + git merge master nightly + git push + Prepare: + needs: [ Merge ] runs-on: [self-hosted, Linux, images] if: ${{ github.repository_owner == 'armbian' }} outputs: @@ -70,13 +110,15 @@ jobs: # clean leftovers sudo rm -rf temp/* cat build/config/targets-cli-beta.conf build/config/targets-desktop-beta.conf | grep -v "^$" | grep -v "^#" | shuf > temp/split.conf - split -d --numeric=1 --number=r/22 --additional-suffix=.conf --suffix-length=2 temp/split.conf temp/split- + PARTS=$(cat temp/split.conf | wc -l) + split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 temp/split.conf temp/split- echo $CHUNKS - name: Prepare matrix id: list_dirs run: - echo ::set-output name=matrix::$(for x in $(seq -w 01 22); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) + PARTS=$(cat temp/split.conf | wc -l) + echo ::set-output name=matrix::$(for x in $(seq -w 1 $(cat temp/split.conf | wc -l)); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) - name: Cache build configurations uses: actions/cache@v2 @@ -119,6 +161,10 @@ jobs: sudo pkill arm-binfmt-P || true sudo pkill aarch64-binfmt-P || true sudo pkill pixz || true + if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then + sudo mountpoint -q build/cache/rootfs && sudo fusermount -u build/cache/rootfs || true + sudo mountpoint -q build/cache/toolchain && sudo fusermount -u build/cache/toolchain || true + fi sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break echo "Mounted temp directories. Trying to unmount." @@ -173,16 +219,20 @@ jobs: run: | - CHUNK="${{ matrix.node }}" + CHUNK="${{ matrix.node }}" + CHUNK=$(echo $CHUNK | sed 's/^0*//') + CHUNK=$(printf "%03d" $CHUNK) cd build # use prepared configs sudo mkdir -p userpatches sudo cp ../scripts/configs/* userpatches/ # prepare host [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes - ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host' - + sudo rm -rf cache/sources + [[ ! -d cache/toolchain ]] && ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host' + sudo rm -rf cache/sources if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then + sudo rm -rf cache/toolchain/* sudo mkdir -p cache/toolchain build/cache/rootfs || true ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true @@ -191,7 +241,9 @@ jobs: fi # calculate how many images we can build in parallel - PARALLEL_BUILDS=$(awk '{printf("%d",$1/5000)}' <<<$(($(LC_ALL=C free -w 2>/dev/null | grep "^Mem" | awk '{print $2}' || LC_ALL=C free | grep "^Mem"| awk '{print $2}')/1024))) + # PARALLEL_BUILDS=$(awk '{printf("%d",$1/5000)}' <<<$(($(LC_ALL=C free -w 2>/dev/null | grep "^Mem" | awk '{print $2}' || LC_ALL=C free | grep "^Mem"| awk '{print $2}')/1024))) + # disable parallel builds since we have lots of runners + PARALLEL_BUILDS=0 # mount deploy target sudo apt-get -y -qq install sshfs sudo mkdir -p /root/.ssh/ diff --git a/.github/workflows/build-beta-kernel.yml b/.github/workflows/build-beta-kernel.yml index 764ddc3d7..2f98256cd 100644 --- a/.github/workflows/build-beta-kernel.yml +++ b/.github/workflows/build-beta-kernel.yml @@ -261,8 +261,8 @@ jobs: BOARD=$(echo $CHUNK | cut -d":" -f1) BRANCH=$(echo $CHUNK | cut -d":" -f2) echo "FILE_NAME=${BOARD}-${BRANCH}" >> $GITHUB_ENV - cd build - sudo rsync -ar --delete ../scripts/hash${FILE_EXT}/. cache/hash${FILE_EXT}/ 2> /dev/null + cd build + sudo rm -f cache/hash${FILE_EXT}/* [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" REPOSITORY_INSTALL="u-boot,kernel" 'prepare_host_basic' if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then @@ -270,17 +270,15 @@ jobs: ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true fi - rm -rf output/debs* + sudo rm -rf output/debs* ./compile.sh ARMBIAN_MIRROR="https://github.com/armbian/mirror/releases/download/" BOARD="$BOARD" CLEAN_LEVEL="alldebs" \ PRIVATE_CCACHE="yes" BETA="$BETA" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no" - [[ $? -eq 0 ]] && echo "UPLOAD=true" >> $GITHUB_ENV - sudo mkdir -p ../build-kernel - echo "${FILE_EXT}" - RAZLIKA=$(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2) - [[ -n $RAZLIKA ]] && cp $RAZLIKA ../build-kernel/ - cp $(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2) ../build-kernel/ - echo "FILE_HASH=$(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2)" >> $GITHUB_ENV - + if [[ $? -eq 0 ]]; then + echo "UPLOAD=true" >> $GITHUB_ENV + sudo mkdir -p ../build-kernel + cp cache/hash${FILE_EXT}/*.git* ../build-kernel/ + echo "FILE_HASH=$(ls -1 cache/hash${FILE_EXT}/*.githash | head -1)" >> $GITHUB_ENV + fi fi - name: Upload hash @@ -370,7 +368,7 @@ jobs: run: | ls -l build-kernel if [[ "$(cat build-kernel/skip 2> /dev/null || true)" == "no" ]]; then - sudo rsync -ar --remove-source-files *.githash scripts/hash${{ env.FILE_EXT }}/ || true + sudo rsync -ar --remove-source-files *.git* scripts/hash${{ env.FILE_EXT }}/ || true cd scripts sudo chown -R $USER:$USER .git if git status --porcelain | grep .; then diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index e33f3e3f0..ceaf61f21 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -5,8 +5,48 @@ on: jobs: - Cache: - + Merge: + + name: "Merge master into nightly" + runs-on: [self-hosted, Linux] + if: ${{ github.repository_owner == 'Armbian' }} + + steps: + + - name: Checkout Armbian build script + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: armbian/build + path: build + ref: nightly + clean: false + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@v3 + with: + gpg-private-key: ${{ secrets.GPG_KEY2 }} + passphrase: ${{ secrets.GPG_PASSPHRASE2 }} + workdir: build + git-user-signingkey: true + git-commit-gpgsign: true + + - name: Merge master into nightly + + run: | + cd build + git config --global user.email "info@armbian.com" + git config --global user.name "Armbianworker" + git checkout master + git fetch + git merge origin/master + git checkout nightly + git merge master nightly + git push + + Prepare: + + needs: [ Merge ] name: Determine needed targets runs-on: [self-hosted, Linux, cache] if: ${{ github.repository_owner == 'armbian' }} @@ -75,30 +115,28 @@ jobs: sudo mkdir -p cache/toolchain cache/rootfs || true ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - fi + fi + sudo rm -f cache/rootfs/*.complete sudo rm -f userpatches/targets.conf - cd ../scripts - ./cacherebuild.sh "" "../build-rootfs/filelist.txt" - cd .. - cd build-rootfs - PARTS=$(cat filelist.txt | wc -l) - split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 filelist.txt split- + ../scripts/cacherebuild.sh "no" "../build-rootfs/filelist.txt" + PARTS=$(cat ../build-rootfs/filelist.txt | wc -l) + split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 ../build-rootfs/filelist.txt ../build-rootfs/split- - name: Prepare matrix id: list_dirs run: | PARTS=$(cat build-rootfs/filelist.txt | wc -l) echo ::set-output name=matrix::$(for x in $(seq -w 1 $PARTS); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) - Job: + Cache: - needs: [ Cache ] + needs: [ Prepare ] runs-on: [self-hosted, Linux, cache] if: ${{ github.repository_owner == 'armbian' }} timeout-minutes: 480 strategy: fail-fast: false matrix: - node: ${{fromJson(needs.Cache.outputs.matrix)}} + node: ${{fromJson(needs.Prepare.outputs.matrix)}} steps: @@ -159,7 +197,7 @@ jobs: workdir: scripts git-user-signingkey: true - - name: Build beta images + - name: Build rootfs cache env: GPG_PASS: ${{ secrets.GPG_PASSPHRASE1 }} @@ -180,7 +218,7 @@ jobs: Finish: name: Finish - needs: [Job] + needs: [Cache] runs-on: [self-hosted, Linux, cache] if: ${{ github.repository_owner == 'Armbian' }} steps: @@ -246,5 +284,5 @@ jobs: ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true fi - sudo rm -f userpatches/targets.conf - ../scripts/cacherebuild.sh "yes" + sudo rm -f userpatches/targets.conf + ../scripts/cacherebuild.sh "yes" test diff --git a/.github/workflows/maintain.yml b/.github/workflows/maintain.yml index b9fc57637..bd3fb355b 100644 --- a/.github/workflows/maintain.yml +++ b/.github/workflows/maintain.yml @@ -69,6 +69,7 @@ jobs: done sudo apt-get -y -qq update sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y -qq upgrade + sudo apt-get purge -y --auto-remove unattended-upgrades [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true cd build @@ -85,184 +86,10 @@ jobs: sudo rm -rf output/debs sudo rm -rf output/debs-beta - Cache: - - name: Make cache - needs: [ Maint ] - runs-on: [self-hosted, Linux, cache] - if: ${{ github.repository_owner == 'armbian' }} - outputs: - matrix: ${{steps.list_dirs.outputs.matrix}} - steps: - - - name: Fix permissions - run: | - # make sure no temporally dirs are mounted from previous runs - while : - do - sudo pkill compile.sh || true - sudo pkill arm-binfmt-P || true - sudo pkill aarch64-binfmt-P || true - sudo pkill pixz || true - sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true - [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 ]] && sudo rm -rf build/.tmp && break - echo "Mounted temp directories. Trying to unmount." - df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true - sleep 10 - done - sudo chown -R $USER:$USER build/.git - - name: Checkout Armbian build script - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/build - path: build - ref: nightly - clean: false - - - name: Checkout Armbian support scripts - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/scripts - token: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} - path: scripts - ref: master - clean: true - - - name: Cache build configurations - uses: actions/cache@v2 - env: - cache-name: build-rootfs - with: - path: build-rootfs - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} - - - name: Build rootfs cache - - run: | - echo "BLTPATH=\"$(pwd)/build/\"" | tee scripts/cacherebuild.conf scripts/betarepository.conf >/dev/null - mkdir -p build-rootfs - rm -f build-rootfs/* - cd build - sudo rm -rf output/images/* - sudo mkdir -p userpatches - sudo cp ../scripts/configs/* userpatches/ - [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes - ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host' - if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then - sudo mkdir -p cache/toolchain cache/rootfs || true - ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - fi - sudo rm -f userpatches/targets.conf - cd ../scripts - ./cacherebuild.sh - cd .. - cd build-rootfs - PARTS=$(($(cat filelist.txt | wc -l)/5)) - split -d --numeric=1 --number=r/32 --additional-suffix=.conf --suffix-length=2 filelist.txt split- - - name: Prepare matrix - id: list_dirs - run: - PARTS=$(($(cat filelist.txt | wc -l)/5)) - echo ::set-output name=matrix::$(for x in $(seq -w 01 32); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq) - - Job: - - needs: [ Cache ] - runs-on: [self-hosted, Linux, cache] - if: ${{ github.repository_owner == 'armbian' }} - timeout-minutes: 480 - strategy: - fail-fast: false - matrix: - node: ${{fromJson(needs.Cache.outputs.matrix)}} - - steps: - - - name: Cache Gradle packages - uses: actions/cache@v2 - env: - cache-name: build-rootfs - with: - path: build-rootfs - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }} - - - name: Fix permissions - run: | - # make sure no temporally dirs are mounted from previous runs - while : - do - sudo pkill compile.sh || true - sudo pkill arm-binfmt-P || true - sudo pkill aarch64-binfmt-P || true - sudo pkill pixz || true - sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true - [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break - echo "Mounted temp directories. Trying to unmount." - df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true - sleep 10 - done - [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true - [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true - - - name: Checkout Armbian build script - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/build - path: build - ref: nightly - clean: false - - - name: Checkout Armbian support scripts - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - repository: armbian/scripts - token: ${{ secrets.SCRIPTS_ACCESS_TOKEN }} - path: scripts - clean: false - - - name: Import GPG key - - uses: crazy-max/ghaction-import-gpg@v3 - with: - gpg-private-key: ${{ secrets.GPG_KEY1 }} - passphrase: ${{ secrets.GPG_PASSPHRASE1 }} - workdir: scripts - git-user-signingkey: true - - - name: Build beta images - env: - GPG_PASS: ${{ secrets.GPG_PASSPHRASE1 }} - - run: | - CHUNK="${{ matrix.node }}" - if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then - sudo mkdir -p build/cache/toolchain build/cache/rootfs || true - ! sudo mountpoint -q build/cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com build/cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - ! sudo mountpoint -q build/cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs build/cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true - fi - cd build - # use prepared configs - sudo mkdir -p userpatches - sudo cp ../scripts/configs/* userpatches/ - # prepare host - [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes || true - bash ../build-rootfs/split-${CHUNK}.conf - Caches: name: Sync & test cache integrity - needs: [Job] + needs: [Maint] runs-on: [self-hosted, Linux, small] if: ${{ github.repository_owner == 'Armbian' }} outputs: @@ -343,7 +170,7 @@ jobs: steps: - name: Fix permissions run: | - sudo chown -R $USER:$USER . + [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true - name: Checkout Armbian build script uses: actions/checkout@v2 with: