From 2e594375a951ed983b98abf8388cdb05e32918da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Igor=20Pe=C4=8Dovnik?=
 <igorpecovnik@users.noreply.github.com>
Date: Fri, 26 Feb 2021 18:27:21 +0100
Subject: [PATCH] Improve cache and toolchain download (#2665)

* Improve cache and toolchain download

* Cleanup
---
 lib/configuration.sh |  8 +++++++-
 lib/debootstrap.sh   | 12 +++++++++++-
 lib/general.sh       | 20 +++++++++++++-------
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/lib/configuration.sh b/lib/configuration.sh
index b13947d38..b67a75721 100644
--- a/lib/configuration.sh
+++ b/lib/configuration.sh
@@ -497,7 +497,13 @@ if [[ $DOWNLOAD_MIRROR == china ]] ; then
 	UBUNTU_MIRROR='mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/'
 fi
 
-ARMBIAN_MIRROR='https://redirect.armbian.com'
+# don't use mirrors that throws garbage on 404
+while true; do
+
+	ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}')
+	[[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break
+
+done
 
 # For user override
 if [[ -f $USERPATCHES_PATH/lib.config ]]; then
diff --git a/lib/debootstrap.sh b/lib/debootstrap.sh
index b43687c04..227940f92 100644
--- a/lib/debootstrap.sh
+++ b/lib/debootstrap.sh
@@ -125,6 +125,16 @@ create_rootfs_cache()
 			download_and_verify "_rootfs" "$cache_name"
 		fi
 
+		if [[ -f $cache_fname && -f $cache_fname.aria2 && $USE_TORRENT="no" && -z "$ROOT_FS_CREATE_ONLY" ]]; then
+			rm ${cache_fname}*
+			download_and_verify "_rootfs" "$cache_name"
+		fi
+
+		if [[ -f $cache_fname.aria2 && -z "$ROOT_FS_CREATE_ONLY" ]]; then
+			display_alert "resuming"
+			download_and_verify "_rootfs" "$cache_name"
+		fi
+
 		if [[ -f $cache_fname ]]; then
 			break
 		else
@@ -133,7 +143,7 @@ create_rootfs_cache()
 
 	done
 
-	if [[ -f $cache_fname && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
+	if [[ -f $cache_fname && ! -f $cache_fname.aria2 && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
 		local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 ))
 		display_alert "Extracting $display_name" "$date_diff days old" "info"
 		pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/
diff --git a/lib/general.sh b/lib/general.sh
index 6ff5255d7..de2fc7796 100644
--- a/lib/general.sh
+++ b/lib/general.sh
@@ -1257,16 +1257,23 @@ download_and_verify()
 
         if [[ $DOWNLOAD_MIRROR == china ]]; then
 		local server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/"
-			else
-		local server="${ARMBIAN_MIRROR}/"
+	else
+		local server=${ARMBIAN_MIRROR}
         fi
 
 	if [[ -f ${localdir}/${dirname}/.download-complete ]]; then
 		return
 	fi
 
+	# switch to china mirror if US timeouts
+	timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
+	if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then
+		display_alert "Timeout from $server" "retrying" "info"
+		server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/"
+	fi
+
 	# check if file exists on remote server before running aria2 downloader
-	[[ ! `wget -S --spider ${server}${remotedir}/${filename}  2>&1 | grep 'HTTP/1.1 200 OK'` ]] && return
+	[[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename}` ]] && return
 
 	cd "${localdir}" || exit
 
@@ -1274,7 +1281,7 @@ download_and_verify()
 	if [[ -f "${SRC}"/config/torrents/${filename}.asc ]]; then
 		local torrent="${SRC}"/config/torrents/${filename}.torrent
 		ln -sf "${SRC}/config/torrents/${filename}.asc" "${localdir}/${filename}.asc"
-	elif [[ ! $(wget -S --spider "${server}${remotedir}/${filename}.asc" 2>&1 >/dev/null | grep 'HTTP/1.1 200 OK') ]]; then
+	elif [[ ! `timeout 10 curl --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then
 		return
 	else
 		# download control file
@@ -1288,7 +1295,7 @@ download_and_verify()
 	if [[ ${USE_TORRENT} == "yes" ]]; then
 
 		display_alert "downloading using torrent network" "$filename"
-		local ariatorrent="--summary-interval=0 --auto-save-interval=0 --seed-time=0 --bt-stop-timeout=15 --console-log-level=error \
+		local ariatorrent="--summary-interval=0 --auto-save-interval=0 --seed-time=0 --bt-stop-timeout=120 --console-log-level=error \
 		--allow-overwrite=true --download-result=hide --rpc-save-upload-metadata=false --auto-file-renaming=false \
 		--file-allocation=trunc --continue=true ${torrent} \
 		--dht-file-path=${SRC}/cache/.aria2/dht.dat --disable-ipv6=true --stderr --follow-torrent=mem --dir=$localdir"
@@ -1309,8 +1316,7 @@ download_and_verify()
 
 	# direct download if torrent fails
 	if [[ ! -f "${localdir}/${filename}.complete" ]]; then
-		if [[ $(wget -S --spider "${server}${remotedir}/${filename}" 2>&1 >/dev/null \
-			| grep 'HTTP/1.1 200 OK') ]]; then
+		if [[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then
 			display_alert "downloading from $(echo $server | cut -d'/' -f3 | cut -d':' -f1) using http(s) network" "$filename"
 			aria2c --download-result=hide --rpc-save-upload-metadata=false --console-log-level=error \
 			--dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=true --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${remotedir}/${filename}") -o "${filename}"