From d3997fe64c5abb4eda9da613a46de6bc9b2387fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Pe=C4=8Dovnik?= Date: Wed, 5 May 2021 20:49:25 +0200 Subject: [PATCH] Cache creation improvements (#2806) --- lib/configuration.sh | 10 ++++++---- lib/debootstrap.sh | 33 +++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/lib/configuration.sh b/lib/configuration.sh index 4ec79377c..15269a541 100644 --- a/lib/configuration.sh +++ b/lib/configuration.sh @@ -525,12 +525,14 @@ if [[ $DOWNLOAD_MIRROR == "bfsu" ]] ; then fi # don't use mirrors that throws garbage on 404 -while true; do +if [[ -z ${ARMBIAN_MIRROR} ]]; then + while true; do - ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}' | head -1) - [[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break + ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}' | head -1) + [[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break -done + done +fi # For user override if [[ -f $USERPATCHES_PATH/lib.config ]]; then diff --git a/lib/debootstrap.sh b/lib/debootstrap.sh index 9a8923cdd..15b538a90 100644 --- a/lib/debootstrap.sh +++ b/lib/debootstrap.sh @@ -104,6 +104,7 @@ create_rootfs_cache() else local cycles=2 fi + # seek last cache, proceed to previous otherwise build it for ((n=0;n<${cycles};n++)); do @@ -118,24 +119,22 @@ create_rootfs_cache() local cache_fname=${SRC}/cache/rootfs/${cache_name} local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.lz4 - display_alert "Checking for local cache" "$display_name" "info" - - if [[ ! -f $cache_fname && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then - display_alert "searching on servers" - download_and_verify "_rootfs" "$cache_name" - fi - - if [[ -f $cache_fname && -f $cache_fname.aria2 && $USE_TORRENT="no" && -z "$ROOT_FS_CREATE_ONLY" ]]; then + if [[ -f $cache_fname && -f $cache_fname.aria2 ]]; then rm ${cache_fname}* download_and_verify "_rootfs" "$cache_name" fi - if [[ -f $cache_fname.aria2 && -z "$ROOT_FS_CREATE_ONLY" ]]; then - display_alert "resuming" + display_alert "Checking for local cache" "$display_name" "info" + if [[ -f $cache_fname && -n "$ROOT_FS_CREATE_ONLY" ]]; then + touch $cache_fname.current + break + else + display_alert "searching on servers" download_and_verify "_rootfs" "$cache_name" fi if [[ -f $cache_fname ]]; then + touch $cache_fname.current break else display_alert "not found: try to use previous cache" @@ -144,6 +143,17 @@ create_rootfs_cache() done if [[ -f $cache_fname && ! -f $cache_fname.aria2 && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then + + # speed up checking + if [[ -n "$ROOT_FS_CREATE_ONLY" ]]; then + touch $cache_fname.current + [[ $use_tmpfs = yes ]] && umount $SDCARD + rm -rf $SDCARD + # remove exit trap + trap - INT TERM EXIT + exit + fi + local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 )) display_alert "Extracting $display_name" "$date_diff days old" "info" pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/ @@ -347,6 +357,9 @@ create_rootfs_cache() echo $GPG_PASS | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes $cache_fname fi + # needed for backend to keep current only + touch $cache_fname.current + fi # used for internal purposes. Faster rootfs cache rebuilding