Update Github actions scripts (#3209)

.github/labeler.yml

@@ -24,6 +24,12 @@
   - config/kernel/*
   - config/sources/*
 
+"c:bsp":
+  - config/sources/**/*
+  - config/boards/*
+  - config/bootscripts/*
+  - config/bootenv/*
+
 "c:build script":
   - lib/*
 
@@ -33,5 +39,5 @@
 "c:cli":
   - config/cli/*
 
-"c:ci":
-  - .github/*
+"c:actions":
+  - .github/workflows/*

.github/workflows/README.md

@@ -5,7 +5,7 @@ Common tags:
 - self-hosted
 - Linux
 - X64
-- ARM64
+- ARM64 (4Gb memory with ZRAM_PERCENTAGE=50)
 - public (isolated runners for merge reqeusts)
 - local (local network)
 - cache (mounted cache)

.github/workflows/build-beta-images.yml

@@ -13,8 +13,48 @@ on:
 #      - completed
 jobs:
 
+  Merge:
+
+    name: "Merge master into nightly"
+    runs-on: [self-hosted, Linux]
+    if: ${{ github.repository_owner == 'Armbian' }}
+
+    steps:
+
+      - name: Checkout Armbian build script
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          repository: armbian/build
+          path: build
+          ref: nightly
+          clean: false
+
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v3
+        with:
+          gpg-private-key: ${{ secrets.GPG_KEY2 }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE2 }}
+          workdir: build
+          git-user-signingkey: true
+          git-commit-gpgsign: true
+
+      - name: Merge master into nightly
+
+        run: |
+          cd build
+          git config --global user.email "info@armbian.com"
+          git config --global user.name "Armbianworker"
+          git checkout master
+          git fetch
+          git merge origin/master
+          git checkout nightly
+          git merge master nightly
+          git push
+
   Prepare:
 
+    needs: [ Merge ]
     runs-on: [self-hosted, Linux, images]
     if: ${{ github.repository_owner == 'armbian' }}
     outputs:
@@ -70,13 +110,15 @@ jobs:
           # clean leftovers
           sudo rm -rf temp/*
           cat build/config/targets-cli-beta.conf build/config/targets-desktop-beta.conf | grep -v "^$" | grep -v "^#" | shuf > temp/split.conf
-          split -d --numeric=1 --number=r/22 --additional-suffix=.conf --suffix-length=2 temp/split.conf temp/split-
+          PARTS=$(cat temp/split.conf | wc -l)
+          split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 temp/split.conf temp/split-
           echo $CHUNKS
 
       - name: Prepare matrix
         id: list_dirs
         run:
-          echo ::set-output name=matrix::$(for x in $(seq -w 01 22); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)
+          PARTS=$(cat temp/split.conf | wc -l)
+          echo ::set-output name=matrix::$(for x in $(seq -w 1 $(cat temp/split.conf | wc -l)); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)          
 
       - name: Cache build configurations
         uses: actions/cache@v2
@@ -119,6 +161,10 @@ jobs:
               sudo pkill arm-binfmt-P || true
               sudo pkill aarch64-binfmt-P || true
               sudo pkill pixz || true
+              if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
+                  sudo mountpoint -q build/cache/rootfs &&  sudo fusermount -u build/cache/rootfs || true
+                  sudo mountpoint -q build/cache/toolchain &&  sudo fusermount -u build/cache/toolchain || true
+              fi
               sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true
               [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break
               echo "Mounted temp directories. Trying to unmount."
@@ -174,15 +220,19 @@ jobs:
         run: |
 
           CHUNK="${{ matrix.node }}"
+          CHUNK=$(echo $CHUNK | sed 's/^0*//') 
+          CHUNK=$(printf "%03d" $CHUNK)
           cd build
           # use prepared configs
           sudo mkdir -p userpatches
           sudo cp ../scripts/configs/* userpatches/
           # prepare host
           [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes
-          ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host'
-          
+          sudo rm -rf cache/sources
+          [[ ! -d cache/toolchain ]] && ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host'
+          sudo rm -rf cache/sources
           if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
+              sudo rm -rf cache/toolchain/*
               sudo mkdir -p cache/toolchain build/cache/rootfs || true
               ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
               ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
@@ -191,7 +241,9 @@ jobs:
           fi
 
           # calculate how many images we can build in parallel
-          PARALLEL_BUILDS=$(awk '{printf("%d",$1/5000)}' <<<$(($(LC_ALL=C free -w 2>/dev/null | grep "^Mem" | awk '{print $2}' || LC_ALL=C free | grep "^Mem"| awk '{print $2}')/1024)))
+          # PARALLEL_BUILDS=$(awk '{printf("%d",$1/5000)}' <<<$(($(LC_ALL=C free -w 2>/dev/null | grep "^Mem" | awk '{print $2}' || LC_ALL=C free | grep "^Mem"| awk '{print $2}')/1024)))
+          # disable parallel builds since we have lots of runners
+          PARALLEL_BUILDS=0
           # mount deploy target
           sudo apt-get -y -qq install sshfs
           sudo mkdir -p /root/.ssh/

.github/workflows/build-beta-kernel.yml

@@ -262,7 +262,7 @@ jobs:
               BRANCH=$(echo $CHUNK | cut -d":" -f2)
               echo "FILE_NAME=${BOARD}-${BRANCH}" >> $GITHUB_ENV
               cd build
-              sudo rsync -ar --delete ../scripts/hash${FILE_EXT}/. cache/hash${FILE_EXT}/ 2> /dev/null
+              sudo rm -f cache/hash${FILE_EXT}/*
               [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes          
               ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" REPOSITORY_INSTALL="u-boot,kernel" 'prepare_host_basic'          
               if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
@@ -270,17 +270,15 @@ jobs:
                   ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
                   ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
               fi 
-              rm -rf output/debs*
+              sudo rm -rf output/debs*
               ./compile.sh ARMBIAN_MIRROR="https://github.com/armbian/mirror/releases/download/" BOARD="$BOARD" CLEAN_LEVEL="alldebs" \
               PRIVATE_CCACHE="yes" BETA="$BETA" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no"
-              [[ $? -eq 0 ]] && echo "UPLOAD=true" >> $GITHUB_ENV
-              sudo mkdir -p ../build-kernel
-              echo "${FILE_EXT}"
-              RAZLIKA=$(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2)
-              [[ -n $RAZLIKA ]] && cp $RAZLIKA ../build-kernel/
-              cp $(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2) ../build-kernel/
-              echo "FILE_HASH=$(diff -q "cache/hash${FILE_EXT}/" "../scripts/hash${FILE_EXT}/" | grep -v gitlog | cut -d" " -f2)"  >> $GITHUB_ENV           
-
+              if [[ $? -eq 0 ]]; then
+                  echo "UPLOAD=true" >> $GITHUB_ENV
+                  sudo mkdir -p ../build-kernel
+                  cp cache/hash${FILE_EXT}/*.git* ../build-kernel/
+                  echo "FILE_HASH=$(ls -1 cache/hash${FILE_EXT}/*.githash | head -1)"  >> $GITHUB_ENV
+              fi
           fi
 
       - name: Upload hash
@@ -370,7 +368,7 @@ jobs:
         run: |
           ls -l build-kernel
           if [[ "$(cat build-kernel/skip 2> /dev/null || true)" == "no" ]]; then
-          sudo rsync -ar --remove-source-files *.githash scripts/hash${{ env.FILE_EXT }}/ || true
+          sudo rsync -ar --remove-source-files *.git* scripts/hash${{ env.FILE_EXT }}/ || true
           cd scripts
           sudo chown -R $USER:$USER .git
           if git status --porcelain | grep .; then

.github/workflows/build-cache.yml

@@ -5,8 +5,48 @@ on:
 
 jobs:
 
-  Cache:
+  Merge:
 
+    name: "Merge master into nightly"
+    runs-on: [self-hosted, Linux]
+    if: ${{ github.repository_owner == 'Armbian' }}
+
+    steps:
+
+      - name: Checkout Armbian build script
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          repository: armbian/build
+          path: build
+          ref: nightly
+          clean: false
+
+      - name: Import GPG key
+        uses: crazy-max/ghaction-import-gpg@v3
+        with:
+          gpg-private-key: ${{ secrets.GPG_KEY2 }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE2 }}
+          workdir: build
+          git-user-signingkey: true
+          git-commit-gpgsign: true
+
+      - name: Merge master into nightly
+
+        run: |
+          cd build
+          git config --global user.email "info@armbian.com"
+          git config --global user.name "Armbianworker"
+          git checkout master
+          git fetch
+          git merge origin/master
+          git checkout nightly
+          git merge master nightly
+          git push
+
+  Prepare:
+
+    needs: [ Merge ]
     name: Determine needed targets
     runs-on: [self-hosted, Linux, cache]
     if: ${{ github.repository_owner == 'armbian' }}
@@ -76,29 +116,27 @@ jobs:
                 ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
                 ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
             fi
+            sudo rm -f cache/rootfs/*.complete
             sudo rm -f userpatches/targets.conf
-            cd ../scripts
-            ./cacherebuild.sh  "" "../build-rootfs/filelist.txt"
-            cd ..
-            cd build-rootfs
-            PARTS=$(cat filelist.txt | wc -l)
-            split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 filelist.txt split-
+            ../scripts/cacherebuild.sh "no" "../build-rootfs/filelist.txt" 
+            PARTS=$(cat ../build-rootfs/filelist.txt | wc -l)
+            split -d --numeric=1 --number=r/${PARTS} --additional-suffix=.conf --suffix-length=3 ../build-rootfs/filelist.txt ../build-rootfs/split-
         - name: Prepare matrix
           id: list_dirs
           run: |
             PARTS=$(cat build-rootfs/filelist.txt | wc -l)
             echo ::set-output name=matrix::$(for x in $(seq -w 1 $PARTS); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)
 
-  Job:
+  Cache:
 
-    needs: [ Cache ]
+    needs: [ Prepare ]
     runs-on: [self-hosted, Linux, cache]
     if: ${{ github.repository_owner == 'armbian' }}
     timeout-minutes: 480
     strategy:
       fail-fast: false
       matrix: 
-        node: ${{fromJson(needs.Cache.outputs.matrix)}}
+        node: ${{fromJson(needs.Prepare.outputs.matrix)}}
 
     steps:
  
@@ -159,7 +197,7 @@ jobs:
           workdir: scripts
           git-user-signingkey: true
 
-      - name: Build beta images
+      - name: Build rootfs cache
         env:
           GPG_PASS: ${{ secrets.GPG_PASSPHRASE1 }}
 
@@ -180,7 +218,7 @@ jobs:
           
   Finish:
     name: Finish
-    needs: [Job]
+    needs: [Cache]
     runs-on: [self-hosted, Linux, cache]
     if: ${{ github.repository_owner == 'Armbian' }}
     steps:
@@ -247,4 +285,4 @@ jobs:
                 ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
             fi  
             sudo rm -f userpatches/targets.conf
-            ../scripts/cacherebuild.sh "yes"
+            ../scripts/cacherebuild.sh "yes" test 

.github/workflows/maintain.yml

@@ -69,6 +69,7 @@ jobs:
           done
           sudo apt-get -y -qq update          
           sudo apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y -qq upgrade
+          sudo apt-get purge -y --auto-remove unattended-upgrades
           [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true
           [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true
           cd build
@@ -85,184 +86,10 @@ jobs:
           sudo rm -rf output/debs
           sudo rm -rf output/debs-beta          
 
-  Cache:
-    
-    name: Make cache
-    needs: [ Maint ]
-    runs-on: [self-hosted, Linux, cache]
-    if: ${{ github.repository_owner == 'armbian' }}
-    outputs:
-      matrix: ${{steps.list_dirs.outputs.matrix}}
-    steps:
-
-        - name: Fix permissions
-          run: |
-            # make sure no temporally dirs are mounted from previous runs
-            while :
-            do
-                sudo pkill compile.sh || true
-                sudo pkill arm-binfmt-P || true
-                sudo pkill aarch64-binfmt-P || true
-                sudo pkill pixz || true
-                sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true
-                [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 ]] && sudo rm -rf build/.tmp && break                
-                echo "Mounted temp directories. Trying to unmount."
-                df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true
-                sleep 10
-            done
-            sudo chown -R $USER:$USER build/.git
-        - name: Checkout Armbian build script
-
-          uses: actions/checkout@v2
-          with:
-            fetch-depth: 0
-            repository: armbian/build
-            path: build
-            ref: nightly
-            clean: false
-
-        - name: Checkout Armbian support scripts
-
-          uses: actions/checkout@v2
-          with:
-            fetch-depth: 0
-            repository: armbian/scripts
-            token: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
-            path: scripts
-            ref: master
-            clean: true
-
-        - name: Cache build configurations
-          uses: actions/cache@v2
-          env:
-            cache-name: build-rootfs
-          with:
-            path: build-rootfs
-            key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-
-        - name: Build rootfs cache
-
-          run: |
-            echo "BLTPATH=\"$(pwd)/build/\"" | tee scripts/cacherebuild.conf scripts/betarepository.conf >/dev/null
-            mkdir -p build-rootfs
-            rm -f build-rootfs/*
-            cd build
-            sudo rm -rf output/images/*
-            sudo mkdir -p userpatches
-            sudo cp ../scripts/configs/* userpatches/
-            [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes
-            ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" USE_TORRENT="yes" REPOSITORY_INSTALL="kernel" 'prepare_host'
-            if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
-                sudo mkdir -p cache/toolchain cache/rootfs || true
-                ! sudo mountpoint -q cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-                ! sudo mountpoint -q cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-            fi  
-            sudo rm -f userpatches/targets.conf
-            cd ../scripts
-            ./cacherebuild.sh
-            cd ..
-            cd build-rootfs
-            PARTS=$(($(cat filelist.txt | wc -l)/5))
-            split -d --numeric=1 --number=r/32 --additional-suffix=.conf --suffix-length=2 filelist.txt split-
-        - name: Prepare matrix
-          id: list_dirs
-          run:
-            PARTS=$(($(cat filelist.txt | wc -l)/5))
-            echo ::set-output name=matrix::$(for x in $(seq -w 01 32); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)
-
-  Job:
-
-    needs: [ Cache ]
-    runs-on: [self-hosted, Linux, cache]
-    if: ${{ github.repository_owner == 'armbian' }}
-    timeout-minutes: 480
-    strategy:
-      fail-fast: false
-      matrix: 
-        node: ${{fromJson(needs.Cache.outputs.matrix)}}
-
-    steps:
- 
-      - name: Cache Gradle packages
-        uses: actions/cache@v2
-        env:
-          cache-name: build-rootfs
-        with:
-          path: build-rootfs
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-
-      - name: Fix permissions
-        run: |
-          # make sure no temporally dirs are mounted from previous runs
-          while :
-          do
-              sudo pkill compile.sh || true
-              sudo pkill arm-binfmt-P || true
-              sudo pkill aarch64-binfmt-P || true
-              sudo pkill pixz || true
-              sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true
-              [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break
-              echo "Mounted temp directories. Trying to unmount."
-              df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true
-              sleep 10
-          done
-          [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true
-          [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true
-
-      - name: Checkout Armbian build script
-
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/build
-          path: build
-          ref: nightly
-          clean: false
-
-      - name: Checkout Armbian support scripts
-
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          repository: armbian/scripts
-          token: ${{ secrets.SCRIPTS_ACCESS_TOKEN }}
-          path: scripts
-          clean: false
-
-      - name: Import GPG key
-
-        uses: crazy-max/ghaction-import-gpg@v3
-        with:
-          gpg-private-key: ${{ secrets.GPG_KEY1 }}
-          passphrase: ${{ secrets.GPG_PASSPHRASE1 }}
-          workdir: scripts
-          git-user-signingkey: true
-
-      - name: Build beta images
-        env:
-          GPG_PASS: ${{ secrets.GPG_PASSPHRASE1 }}
-
-        run: |
-          CHUNK="${{ matrix.node }}"
-          if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
-              sudo mkdir -p build/cache/toolchain build/cache/rootfs || true
-              ! sudo mountpoint -q build/cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com build/cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-              ! sudo mountpoint -q build/cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs build/cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
-          fi          
-          cd build
-          # use prepared configs
-          sudo mkdir -p userpatches
-          sudo cp ../scripts/configs/* userpatches/
-          # prepare host
-          [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes || true
-          bash ../build-rootfs/split-${CHUNK}.conf
-
   Caches:
 
     name: Sync & test cache integrity
-    needs: [Job]
+    needs: [Maint]
     runs-on: [self-hosted, Linux, small]
     if: ${{ github.repository_owner == 'Armbian' }}
     outputs:
@@ -343,7 +170,7 @@ jobs:
     steps:
       - name: Fix permissions
         run: |
-          sudo chown -R $USER:$USER .
+          [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true          
       - name: Checkout Armbian build script
         uses: actions/checkout@v2
         with: