JDM170/build
1
0
Fork 0
mirror of https://github.com/armbian/build synced 2025-09-24 19:47:06 +07:00
Code Issues Packages Projects Releases Wiki Activity

Improve cache and toolchain download (#2665)

Browse Source 
* Improve cache and toolchain download

* Cleanup
This commit is contained in:
Igor Pečovnik
2021-02-26 18:27:21 +01:00
committed by GitHub
parent 9dd16212e1
commit 2e594375a9
3 changed files with 31 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/configuration.sh
View File

@@ -497,7 +497,13 @@ if [[ $DOWNLOAD_MIRROR == china ]] ; then
UBUNTU_MIRROR='mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/'
fi
ARMBIAN_MIRROR='https://redirect.armbian.com'
# don't use mirrors that throws garbage on 404
while true; do
ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}')
[[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]] && break
done
# For user override
if [[ -f $USERPATCHES_PATH/lib.config ]]; then

12
lib/debootstrap.sh
View File

@@ -125,6 +125,16 @@ create_rootfs_cache()
download_and_verify "_rootfs" "$cache_name"
fi
if [[ -f $cache_fname && -f $cache_fname.aria2 && $USE_TORRENT="no" && -z "$ROOT_FS_CREATE_ONLY" ]]; then
rm ${cache_fname}*
download_and_verify "_rootfs" "$cache_name"
fi
if [[ -f $cache_fname.aria2 && -z "$ROOT_FS_CREATE_ONLY" ]]; then
display_alert "resuming"
download_and_verify "_rootfs" "$cache_name"
fi
if [[ -f $cache_fname ]]; then
break
else
@@ -133,7 +143,7 @@ create_rootfs_cache()
done
if [[ -f $cache_fname && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
if [[ -f $cache_fname && ! -f $cache_fname.aria2 && "$ROOT_FS_CREATE_ONLY" != "force" ]]; then
local date_diff=$(( ($(date +%s) - $(stat -c %Y $cache_fname)) / 86400 ))
display_alert "Extracting $display_name" "$date_diff days old" "info"
pv -p -b -r -c -N "[ .... ] $display_name" "$cache_fname" | lz4 -dc | tar xp --xattrs -C $SDCARD/

20
lib/general.sh
View File

@@ -1257,16 +1257,23 @@ download_and_verify()
if [[ $DOWNLOAD_MIRROR == china ]]; then
local server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/"
else
local server="${ARMBIAN_MIRROR}/"
else
local server=${ARMBIAN_MIRROR}
fi
if [[ -f ${localdir}/${dirname}/.download-complete ]]; then
return
fi
# switch to china mirror if US timeouts
timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null
if [[ $? -ne 7 && $? -ne 22 && $? -ne 0 ]]; then
display_alert "Timeout from $server" "retrying" "info"
server="https://mirrors.tuna.tsinghua.edu.cn/armbian-releases/"
fi
# check if file exists on remote server before running aria2 downloader
[[ ! `wget -S --spider ${server}${remotedir}/${filename} 2>&1 | grep 'HTTP/1.1 200 OK'` ]] && return
[[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename}` ]] && return
cd "${localdir}" || exit
@@ -1274,7 +1281,7 @@ download_and_verify()
if [[ -f "${SRC}"/config/torrents/${filename}.asc ]]; then
local torrent="${SRC}"/config/torrents/${filename}.torrent
ln -sf "${SRC}/config/torrents/${filename}.asc" "${localdir}/${filename}.asc"
elif [[ ! $(wget -S --spider "${server}${remotedir}/${filename}.asc" 2>&1 >/dev/null | grep 'HTTP/1.1 200 OK') ]]; then
elif [[ ! `timeout 10 curl --head --fail --silent "${server}${remotedir}/${filename}.asc"` ]]; then
return
else
# download control file
@@ -1288,7 +1295,7 @@ download_and_verify()
if [[ ${USE_TORRENT} == "yes" ]]; then
display_alert "downloading using torrent network" "$filename"
local ariatorrent="--summary-interval=0 --auto-save-interval=0 --seed-time=0 --bt-stop-timeout=15 --console-log-level=error \
local ariatorrent="--summary-interval=0 --auto-save-interval=0 --seed-time=0 --bt-stop-timeout=120 --console-log-level=error \
--allow-overwrite=true --download-result=hide --rpc-save-upload-metadata=false --auto-file-renaming=false \
--file-allocation=trunc --continue=true ${torrent} \
--dht-file-path=${SRC}/cache/.aria2/dht.dat --disable-ipv6=true --stderr --follow-torrent=mem --dir=$localdir"
@@ -1309,8 +1316,7 @@ download_and_verify()
# direct download if torrent fails
if [[ ! -f "${localdir}/${filename}.complete" ]]; then
if [[ $(wget -S --spider "${server}${remotedir}/${filename}" 2>&1 >/dev/null \
| grep 'HTTP/1.1 200 OK') ]]; then
if [[ ! `timeout 10 curl --head --fail --silent ${server}${remotedir}/${filename} 2>&1 >/dev/null` ]]; then
display_alert "downloading from $(echo $server | cut -d'/' -f3 | cut -d':' -f1) using http(s) network" "$filename"
aria2c --download-result=hide --rpc-save-upload-metadata=false --console-log-level=error \
--dht-file-path="${SRC}"/cache/.aria2/dht.dat --disable-ipv6=true --summary-interval=0 --auto-file-renaming=false --dir="${localdir}" ${server}${remotedir}/${filename} $(webseed "${remotedir}/${filename}") -o "${filename}"