CI pipeline refactoring (#3173)

2025-09-24 19:47:06 +07:00 · 2021-10-06 22:47:23 +02:00
parent 4556ebc0fa
commit 21f13b4ba5
10 changed files with 1067 additions and 868 deletions
--- a/.github/workflows/build-u-boot.yml
+++ b/.github/workflows/build-u-boot.yml
@@ -6,7 +6,8 @@ on:
 jobs:

  Prepare:
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, Linux, small]
+    if: ${{ github.repository_owner == 'armbian' }}
    outputs:
      matrix: ${{steps.list_dirs.outputs.matrix}}
    steps:
@@ -20,23 +21,24 @@ jobs:
          ref: nightly
          clean: false

+      - name: Cache build configurations
+        uses: actions/cache@v2
+        env:
+          cache-name: build-u-boot
+        with:
+          path: build-u-boot
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
+
      - name: Prepare matrix
        id: list_dirs
        run:
          echo ::set-output name=matrix::$(for x in $(cat build/config/target*.conf | grep -v "^$" | grep -v "^#" | awk '{print $1":"$2}' | sort | uniq); do echo $x; done|jq -cnR '[inputs | select(length>0)]' | jq)

-      - name: Cache build configurations
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-build
-        with:
-          path: temp
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-
  U-boot: # short name because Github will expand with the matrix values

    needs: [ Prepare ]
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, Linux, small]
+    if: ${{ github.repository_owner == 'armbian' }}
    timeout-minutes: 480
    strategy:
      fail-fast: false
@@ -44,26 +46,46 @@ jobs:
        node: ${{fromJson(needs.Prepare.outputs.matrix)}}

    steps:
- 
-      - name: Cache Gradle packages
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-build
-        with:
-          path: temp
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}

      - name: Install SSH key for storage
-
        uses: shimataro/ssh-key-action@v2
        with:
          key: ${{ secrets.KEY_TORRENTS }}
          known_hosts: ${{ secrets.KNOWN_HOSTS_UPLOAD }}
          if_key_exists: replace

+      - name: Fix permissions
+        run: |
+
+          # make sure no temporally dirs are mounted from previous runs
+          while :
+          do
+              sudo pkill compile.sh || true
+              sudo pkill arm-binfmt-P || true
+              sudo pkill aarch64-binfmt-P || true
+              sudo pkill pixz || true
+              sudo mountpoint -q build/output/images && sudo fusermount -u build/output/images || true
+              [[ "$(df | grep "/.tmp" | wc -l)" -eq 0 && $(sudo mountpoint -q build/output/images; echo $?) -eq 1 ]] && sudo rm -rf build/.tmp && break
+              echo "Mounted temp directories. Trying to unmount."
+              df | grep ".tmp" | awk '{print $6}' | xargs sudo umount 2>/dev/null || true
+              sleep 10
+          done
+          [[ -d build/.git ]] && sudo chown -R $USER:$USER build/.git || true
+          [[ -d build/output/images ]] && sudo rm -rf build/output/images/* || true
+ 
+ 
+      - name: Cache build configurations
+        uses: actions/cache@v2
+        env:
+          cache-name: build-u-boot
+        with:
+          path: build-u-boot
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-${{ github.run_id }}
+ 
      - name: Checkout Armbian build script
+
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
@@ -76,22 +98,32 @@ jobs:
        run: |
          CHUNK="${{ matrix.node }}"
          BOARD=$(echo $CHUNK | cut -d":" -f1)
+          echo "BOARD=$BOARD" >> $GITHUB_ENV
          BRANCH=$(echo $CHUNK | cut -d":" -f2)
+          echo "BRANCH=$BRANCH" >> $GITHUB_ENV
          cd build
          [[ ! -f .ignore_changes ]] && sudo touch .ignore_changes
-          ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" REPOSITORY_INSTALL="u-boot,kernel" 'prepare_host'
-          ./compile.sh BOARD="$BOARD" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no" 'fetch_from_repo "$BOOTSOURCE" "$BOOTDIR" "$BOOTBRANCH" "yes"' 
-          # mount deploy target
-          sudo apt-get -y -qq install sshfs
-          sudo mkdir -p /root/.ssh/
-          sudo cp ~/.ssh/known_hosts /root/.ssh/
-          sudo mkdir -p output/debs-beta || true
-          sudo sshfs upload@users.armbian.com:/debs-beta output/debs-beta -o IdentityFile=~/.ssh/id_rsa
-          ./compile.sh BOARD="$BOARD" BETA="yes" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no" compile_uboot
-          # umount
-          while mountpoint output/images -q
-          do
-              sudo umount output/images || true
-              echo "Trying to unmount ..."
-              sleep 10
-          done
+          sudo rm -rf cache/source/u-boot cache/source/linux-* output/debs/* output/debs-beta/*
+          ./compile.sh KERNEL_ONLY="yes" BOARD="bananapi" BRANCH="current" KERNEL_CONFIGURE="no" REPOSITORY_INSTALL="u-boot,kernel" 'prepare_host_basic'
+           if [[ $(curl -s http://ifconfig.me) == "93.103.15.56" ]]; then
+              sudo mkdir -p build/cache/toolchain build/cache/rootfs || true
+              ! sudo mountpoint -q build/cache/toolchain && sudo mount nas:/tank/armbian/toolchain.armbian.com build/cache/toolchain -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
+              ! sudo mountpoint -q build/cache/rootfs && sudo mount nas:/tank/armbian/dl.armbian.com/_rootfs build/cache/rootfs -o rsize=32768,wsize=32768,timeo=5,retrans=2,actimeo=60,retry=15 || true
+          fi  
+          ./compile.sh ARMBIAN_MIRROR="https://github.com/armbian/mirror/releases/download/" BOARD="$BOARD" PRIVATE_CCACHE="yes" BETA="yes" KERNEL_ONLY="yes" BRANCH="$BRANCH" KERNEL_CONFIGURE="no" OFFLINE="no" REPOSITORY_INSTALL="kernel,bsp,armbian-zsh,armbian-config,armbian-firmware"
+          mkdir -p ../build-u-boot
+          cp output/debs-beta/linux-u-boot-${BRANCH}-${BOARD}_$(cat VERSION)_* ../build-u-boot/
+
+      - name: upload artefacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: u-boot-${{ env.BOARD }}-${{ env.BRANCH }}
+          path: build/output/debs-beta/*u-boot*
+
+      - name: Deploy to server
+        if: ${{ success() }}
+        run: |
+          while fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do sleep 1; done;
+          sudo apt-get -y -qq install lftp
+          sudo chown -R $USER:$USER build/output/debs-beta/
+          lftp -u upload, -e "set net:timeout 4;set net:max-retries 6;mirror --Remove-source-files -R --no-empty-dirs --parallel=8 --no-perms $(pwd)/build/output/debs-beta/ debs-beta/ ;bye" sftp://users.armbian.com