Cache creation improvements (#2806)

This commit is contained in:
Igor Pečovnik
2021-05-05 20:49:25 +02:00
committed by GitHub
parent f43db66d32
commit d3997fe64c
2 changed files with 29 additions and 14 deletions

View File

@@ -525,12 +525,14 @@ if [[ $DOWNLOAD_MIRROR == "bfsu" ]] ; then
fi fi
# don't use mirrors that throws garbage on 404 # don't use mirrors that throws garbage on 404
while true; do if [[ -z ${ARMBIAN_MIRROR} ]]; then
while true; do
ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}' | head -1) ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}' | head -1)
[[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break [[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break
done done
fi
# For user override # For user override
if [[ -f $USERPATCHES_PATH/lib.config ]]; then if [[ -f $USERPATCHES_PATH/lib.config ]]; then

View File

@@ -104,6 +104,7 @@ create_rootfs_cache()
else else
local cycles=2 local cycles=2
fi fi
# seek last cache, proceed to previous otherwise build it # seek last cache, proceed to previous otherwise build it
for ((n=0;n<${cycles};n++)); do for ((n=0;n<${cycles};n++)); do
@@ -118,24 +119,22 @@ create_rootfs_cache()
local cache_fname=${SRC}/cache/rootfs/${cache_name} local cache_fname=${SRC}/cache/rootfs/${cache_name}
local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.lz4 local display_name=${RELEASE}-${cache_type}-${ARCH}.${packages_hash:0:3}...${packages_hash:29}.tar.lz4
display_alert "Checking for local cache" "$display_name" "info" if [[ -f $cache_fname && -f $cache_fname.aria2 ]]; then
if [[ ! -f $cache_fname && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
display_alert "searching on servers"
download_and_verify "_rootfs" "$cache_name"
fi
if [[ -f $cache_fname && -f $cache_fname.aria2 && $USE_TORRENT="no" && -z "$ROOT_FS_CREATE_ONLY" ]]; then
rm ${cache_fname}* rm ${cache_fname}*
download_and_verify "_rootfs" "$cache_name" download_and_verify "_rootfs" "$cache_name"
fi fi
if [[ -f $cache_fname.aria2 && -z "$ROOT_FS_CREATE_ONLY" ]]; then display_alert "Checking for local cache" "$display_name" "info"
display_alert "resuming" if [[ -f $cache_fname && -n "$ROOT_FS_CREATE_ONLY" ]]; then
touch $cache_fname.current
break
else
display_alert "searching on servers"
download_and_verify "_rootfs" "$cache_name" download_and_verify "_rootfs" "$cache_name"
fi fi
if [[ -f $cache_fname ]]; then if [[ -f $cache_fname ]]; then
touch $cache_fname.current
break break
else else
display_alert "not found: try to use previous cache" display_alert "not found: try to use previous cache"
@@ -144,6 +143,17 @@ create_rootfs_cache()
done done
if [[ -f $cache_fname && ! -f $cache_fname.aria2 && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then if [[ -f $cache_fname && ! -f $cache_fname.aria2 && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
# speed up checking
if [[ -n "$ROOT_FS_CREATE_ONLY" ]]; then
touch $cache_fname.current
[[ $use_tmpfs = yes ]] && umount $SDCARD
rm -rf $SDCARD
# remove exit trap
trap - INT TERM EXIT
exit
fi
local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 )) local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 ))
display_alert "Extracting $display_name" "$date_diff days old" "info" display_alert "Extracting $display_name" "$date_diff days old" "info"
pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/ pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/
@@ -347,6 +357,9 @@ create_rootfs_cache()
echo $GPG_PASS | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes $cache_fname echo $GPG_PASS | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes $cache_fname
fi fi
# needed for backend to keep current only
touch $cache_fname.current
fi fi
# used for internal purposes. Faster rootfs cache rebuilding # used for internal purposes. Faster rootfs cache rebuilding