Improve creating images from repository (#2340)

* Few cosmetic changes and updated configs
* Improve repository install
* Update configs
* Adjust configs

lib/build-all-ng.sh

@@ -161,7 +161,7 @@ build_main ()
 		source "${SRC}"/lib/main.sh
 
 	fi
-
+	cd "${SRC}"
 	rm "/run/armbian/Armbian_${BOARD^}_${BRANCH}_${RELEASE}_${BUILD_DESKTOP}_${BUILD_MINIMAL}.pid"
 }
 

lib/debootstrap.sh

@@ -246,8 +246,8 @@ create_rootfs_cache()
 		# DEBUG: print free space
 		local freespace=$(LC_ALL=C df -h)
 		echo $freespace >> $DEST/debug/debootstrap.log
-		display_alert "SD" "$(echo -e "$freespace" | grep $SDCARD | awk '{print $5}')" "info"
-		display_alert "Mount point" "$(echo -e "$freespace" | grep $MOUNT | awk '{print $5}')" "info"
+		display_alert "Free SD cache" "$(echo -e "$freespace" | grep $SDCARD | awk '{print $5}')" "info"
+		display_alert "Mount point" "$(echo -e "$freespace" | grep $MOUNT | head -1 | awk '{print $5}')" "info"
 
 		# create list of installed packages for debug purposes
 		chroot $SDCARD /bin/bash -c "dpkg --get-selections" | grep -v deinstall | awk '{print $1}' | cut -f1 -d':' > ${cache_fname}.list 2>&1
@@ -475,7 +475,7 @@ prepare_partitions()
 
 		check_loop_device "$rootdevice"
 		display_alert "Creating rootfs" "$ROOTFS_TYPE on $rootdevice"
-		mkfs.${mkfs[$ROOTFS_TYPE]} ${mkopts[$ROOTFS_TYPE]} $rootdevice
+		mkfs.${mkfs[$ROOTFS_TYPE]} ${mkopts[$ROOTFS_TYPE]} $rootdevice >> "${DEST}"/debug/install.log 2>&1
 		[[ $ROOTFS_TYPE == ext4 ]] && tune2fs -o journal_data_writeback $rootdevice > /dev/null
 		if [[ $ROOTFS_TYPE == btrfs && $BTRFS_COMPRESSION != none ]]; then
 			local fscreateopt="-o compress-force=${BTRFS_COMPRESSION}"
@@ -492,9 +492,9 @@ prepare_partitions()
 		echo "$rootfs / ${mkfs[$ROOTFS_TYPE]} defaults,noatime,nodiratime${mountopts[$ROOTFS_TYPE]} 0 1" >> $SDCARD/etc/fstab
 	fi
 	if [[ -n $bootpart ]]; then
-		display_alert "Creating /boot" "$bootfs"
+		display_alert "Creating /boot" "$bootfs on ${LOOP}p${bootpart}"
 		check_loop_device "${LOOP}p${bootpart}"
-		mkfs.${mkfs[$bootfs]} ${mkopts[$bootfs]} ${LOOP}p${bootpart}
+		mkfs.${mkfs[$bootfs]} ${mkopts[$bootfs]} ${LOOP}p${bootpart} >> "${DEST}"/debug/install.log 2>&1
 		mkdir -p $MOUNT/boot/
 		mount ${LOOP}p${bootpart} $MOUNT/boot/
 		echo "UUID=$(blkid -s UUID -o value ${LOOP}p${bootpart}) /boot ${mkfs[$bootfs]} defaults${mountopts[$bootfs]} 0 2" >> $SDCARD/etc/fstab
@@ -569,7 +569,7 @@ update_initramfs()
 	mount_chroot "$chroot_target/"
 
 	chroot $chroot_target /bin/bash -c "$update_initramfs_cmd" >> $DEST/debug/install.log 2>&1
-	display_alert "Updated initramfs." "for details see: $DEST/debug/install.log" "ext"
+	display_alert "Updated initramfs." "for details see: $DEST/debug/install.log" "info"
 
 	display_alert "Re-enabling" "initramfs-tools hook for kernel"
 	chroot "${SDCARD}" /bin/bash -c "chmod +x /etc/kernel/postinst.d/initramfs-tools" >> "${DEST}"/debug/install.log 2>&1
@@ -617,8 +617,8 @@ create_image()
 	# DEBUG: print free space
 	local freespace=$(LC_ALL=C df -h)
 	echo $freespace >> $DEST/debug/debootstrap.log
-	display_alert "SD" "$(echo -e "$freespace" | grep $SDCARD | awk '{print $5}')" "info"
-	display_alert "Mount point" "$(echo -e "$freespace" | grep $MOUNT | awk '{print $5}')" "info"
+	display_alert "Free SD cache" "$(echo -e "$freespace" | grep $SDCARD | awk '{print $5}')" "info"
+	display_alert "Mount point" "$(echo -e "$freespace" | grep $MOUNT | head -1 | awk '{print $5}')" "info"
 
 	# stage: write u-boot
 	write_uboot $LOOP

lib/distributions.sh

@@ -210,6 +210,8 @@ install_common()
 	ff02::2     ip6-allrouters
 	EOF
 
+	cd $SRC
+
 	display_alert "Updating" "package lists"
 	chroot "${SDCARD}" /bin/bash -c "apt-get update" >> "${DEST}"/debug/install.log 2>&1
 
@@ -241,13 +243,8 @@ install_common()
 		UBOOT_VER=$(dpkg --info "${DEB_STORAGE}/${CHOSEN_UBOOT}_${REVISION}_${ARCH}.deb" | grep Descr | awk '{print $(NF)}')
 		install_deb_chroot "${DEB_STORAGE}/${CHOSEN_UBOOT}_${REVISION}_${ARCH}.deb"
 	else
-		UBOOT_VER=$(chroot "${SDCARD}" /bin/bash -c "apt-cache --names-only search ^linux-u-boot-${BOARD}-${BRANCH} | awk '{print \$(NF)}'")
-		display_alert "Installing from repository" "linux-u-boot-${BOARD}-${BRANCH} $UBOOT_VER"
-		chroot "${SDCARD}" /bin/bash -c "apt-get -qq -y install linux-u-boot-${BOARD}-${BRANCH}" >> "${DEST}"/debug/install.log 2>&1
-		# we need package later, move to output, apt-get must be here, apt deletes file
-		UPSTREM_VER=$(dpkg-deb -I "${SDCARD}"/var/cache/apt/archives/linux-u-boot-${BOARD}-${BRANCH}*_${ARCH}.deb | grep Version | awk '{print $(NF)}')
-		rsync -rq "${SDCARD}"/var/cache/apt/archives/linux-u-boot-${BOARD}-${BRANCH}*_${ARCH}.deb ${DEB_STORAGE}/${CHOSEN_UBOOT}_${UPSTREM_VER}_${ARCH}.deb
-		[[ $? -ne 0 ]] && exit_with_error "U-boot linux-u-boot-${BOARD}-${BRANCH}*_${ARCH}.deb download failed"
+		install_deb_chroot "linux-u-boot-${BOARD}-${BRANCH}" "remote"
+		UPSTREM_VER=$(dpkg-deb -f "${SDCARD}"/var/cache/apt/archives/linux-u-boot-${BOARD}-${BRANCH}*_${ARCH}.deb Version)
 	fi
 
 	# install kernel
@@ -262,24 +259,19 @@ install_common()
 			install_deb_chroot "${DEB_STORAGE}/${CHOSEN_KERNEL/image/headers}_${REVISION}_${ARCH}.deb"
 		fi
 	else
-		VER=$(chroot "${SDCARD}" /bin/bash -c "apt-cache --names-only search ^linux-image-${BRANCH}-${LINUXFAMILY} | awk '{print \$(NF)}'")
+		install_deb_chroot "linux-image-${BRANCH}-${LINUXFAMILY}" "remote"
+		VER=$(dpkg-deb -f "${SDCARD}"/var/cache/apt/archives/linux-image-${BRANCH}-${LINUXFAMILY}*_${ARCH}.deb Source)
 		VER="${VER/-$LINUXFAMILY/}"
-		display_alert "Installing from repository" "linux-image-${BRANCH}-${LINUXFAMILY} $VER"
-		chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install linux-image-${BRANCH}-${LINUXFAMILY}" >> "${DEST}"/debug/install.log 2>&1
-		display_alert "Installing from repository" "linux-dtb-${BRANCH}-${LINUXFAMILY}"
-		chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install linux-dtb-${BRANCH}-${LINUXFAMILY}" >> "${DEST}"/debug/install.log 2>&1
-		if [[ $INSTALL_HEADERS == yes ]]; then
-			display_alert "Installing from repository" "armbian-headers"
-			chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install linux-headers-${BRANCH}-${LINUXFAMILY}" >> "${DEST}"/debug/install.log 2>&1
-		fi
+		VER="${VER/linux-/}"
+		install_deb_chroot "linux-dtb-${BRANCH}-${LINUXFAMILY}" "remote"
+		[[ $INSTALL_HEADERS == yes ]] && install_deb_chroot "linux-headers-${BRANCH}-${LINUXFAMILY}" "remote"
 	fi
 
 	# install board support packages
 	if [[ "${REPOSITORY_INSTALL}" != *bsp* ]]; then
 		install_deb_chroot "${DEB_STORAGE}/$RELEASE/${CHOSEN_ROOTFS}_${REVISION}_${ARCH}.deb" >> "${DEST}"/debug/install.log 2>&1
 	else
-		display_alert "Installing from repository" "${CHOSEN_ROOTFS}"
-		chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install ${CHOSEN_ROOTFS}" >> "${DEST}"/debug/install.log 2>&1
+		install_deb_chroot "${CHOSEN_ROOTFS}" "remote"
 	fi
 
 	# install armbian-desktop
@@ -291,8 +283,7 @@ install_common()
 		fi
 	else
 		if [[ $BUILD_DESKTOP == yes ]]; then
-			display_alert "Installing from repository" "armbian-${RELEASE}-desktop"
-			chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install armbian-${RELEASE}-desktop" >> "${DEST}"/debug/install.log 2>&1
+			install_deb_chroot "armbian-${RELEASE}-desktop" "remote"
 			# install display manager and PACKAGE_LIST_DESKTOP_FULL packages if enabled per board
 			desktop_postinstall
 		fi
@@ -304,8 +295,7 @@ install_common()
 			install_deb_chroot "${DEB_STORAGE}/armbian-firmware_${REVISION}_all.deb"
 		fi
 	else
-		display_alert "Installing from repository" "armbian-firmware"
-		chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install armbian-firmware" >> "${DEST}"/debug/install.log 2>&1
+		install_deb_chroot "armbian-firmware" "remote"
 	fi
 
 	# install armbian-config
@@ -315,8 +305,7 @@ install_common()
 		fi
 	else
 		if [[ $BUILD_MINIMAL != yes ]]; then
-			display_alert "Installing from repository" "armbian-config"
-			chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install armbian-config" >> "${DEST}"/debug/install.log 2>&1
+			install_deb_chroot "armbian-config" "remote"
 		fi
 	fi
 
@@ -327,7 +316,7 @@ install_common()
 
 	# install wireguard tools
 	if [[ $WIREGUARD == yes ]]; then
-		chroot "${SDCARD}" /bin/bash -c "apt-get -y -qq install wireguard-tools --no-install-recommends" >> "${DEST}"/debug/install.log 2>&1
+		install_deb_chroot "wireguard-tools --no-install-recommends" "remote"
 	fi
 
 	# freeze armbian packages

lib/image-helpers.sh

@@ -94,8 +94,13 @@ write_uboot()
 	TEMP_DIR=$(mktemp -d || exit 1)
 	chmod 700 ${TEMP_DIR}
 	revision=${REVISION}
-	[[ -n $UPSTREM_VER ]] && revision=${UPSTREM_VER}
-	dpkg -x "${DEB_STORAGE}/${CHOSEN_UBOOT}_${revision}_${ARCH}.deb" ${TEMP_DIR}/
+	if [[ -n $UPSTREM_VER ]]; then
+		DEB_BRANCH=${DEB_BRANCH/-/}
+		revision=${UPSTREM_VER}
+		dpkg -x "${DEB_STORAGE}/linux-u-boot-${BOARD}-${DEB_BRANCH/-/}_${revision}_${ARCH}.deb" ${TEMP_DIR}/
+	else
+		dpkg -x "${DEB_STORAGE}/${CHOSEN_UBOOT}_${revision}_${ARCH}.deb" ${TEMP_DIR}/
+	fi
 	write_uboot_platform "${TEMP_DIR}/usr/lib/${CHOSEN_UBOOT}_${revision}_${ARCH}" "$loop"
 	[[ $? -ne 0 ]] && exit_with_error "U-boot bootloader failed to install" "@host"
 	rm -rf ${TEMP_DIR}
@@ -123,11 +128,21 @@ customize_image()
 install_deb_chroot()
 {
 	local package=$1
+	local variant=$2
 	local name
-	name=$(basename "${package}")
-	[[ ! -f "${SDCARD}/root/${name}" ]] && cp "${package}" "${SDCARD}/root/${name}"
-	display_alert "Installing" "$name"
+	local desc
+	if [[ ${variant} != remote ]]; then
+		name="/root/"$(basename "${package}")
+		[[ ! -f "${SDCARD}${name}" ]] && cp "${package}" "${SDCARD}${name}"
+		desc=""
+	else
+		name=$1
+		desc=" from repository"
+	fi
+
+	display_alert "Installing${desc}" "${name/\/root\//}"
 	[[ $NO_APT_CACHER != yes ]] && local apt_extra="-o Acquire::http::Proxy=\"http://${APT_PROXY_ADDR:-localhost:3142}\" -o Acquire::http::Proxy::localhost=\"DIRECT\""
 	LC_ALL=C LANG=C chroot "${SDCARD}" /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get -yqq \
-		$apt_extra --no-install-recommends install ./root/$name" >> "${DEST}"/debug/install.log 2>&1
+		$apt_extra --no-install-recommends install $name" >> "${DEST}"/debug/install.log 2>&1
+	[[ ${variant} == remote ]] && rsync -rq "${SDCARD}"/var/cache/apt/archives/linux-u-boot-${BOARD}-${BRANCH}*_${ARCH}.deb ${DEB_STORAGE}/
 }

lib/main.sh

@@ -509,7 +509,7 @@ $([[ -n $BUILD_DESKTOP ]] && echo "BUILD_DESKTOP=${BUILD_DESKTOP} ")\
 $([[ -n $KERNEL_ONLY ]] && echo "KERNEL_ONLY=${KERNEL_ONLY} ")\
 $([[ -n $KERNEL_CONFIGURE ]] && echo "KERNEL_CONFIGURE=${KERNEL_CONFIGURE} ")\
 $([[ -n $COMPRESS_OUTPUTIMAGE ]] && echo "COMPRESS_OUTPUTIMAGE=${COMPRESS_OUTPUTIMAGE} ")\
-" "info"
+" "ext"
 
 } # end of do_default()