prepare_host: Installing packages with the install_pkg_deb function (#3374)

* separate joint and own lists for architecture

Signed-off-by: The-going <48602507+The-going@users.noreply.github.com>

* Remove duplicate selection

* Final add Installing build dependencies

* libncurses: Do not use transitional package

Signed-off-by: The-going <48602507+The-going@users.noreply.github.com>

* add arch i386: Remove duplicate selection

* lib32ncurses-dev: Dependency only for amd64

* install_pkg_deb: parameter verbose to log

lib/general.sh

@@ -1177,9 +1177,12 @@ wait_for_package_manager()
 
 
 # Installing debian packages in the armbian build system.
+# The function accepts three optional parameters:
+# upgrade, clean - the same name for apt
+# verbose - detailed log for the function
 #
-# list="upgrade clean pkg1 pkg2 pkg3 pkgbadname pkg-1.0 | pkg-2.0 pkg5 (>= 9)"
+# list="pkg1 pkg2 pkg3 pkgbadname pkg-1.0 | pkg-2.0 pkg5 (>= 9)"
-# install_pkg_deb $list
+# install_pkg_deb upgrade verbose $list
 #
 # If the package has a bad name, we will see it in the log file.
 # If there is an LOG_OUTPUT_FILE variable and it has a value as
@@ -1195,6 +1198,7 @@ install_pkg_deb ()
 	local for_install
 	local need_upgrade=false
 	local need_clean=false
+	local need_verbose=false
 	local _line=${BASH_LINENO[0]}
 	local _function=${FUNCNAME[1]}
 	local _file=$(basename "${BASH_SOURCE[1]}")
@@ -1206,6 +1210,7 @@ install_pkg_deb ()
 		case $p in
 			upgrade) need_upgrade=true; continue ;;
 			clean) need_clean=true; continue ;;
+			verbose) need_verbose=true; continue ;;
 			\||\(*|*\)) continue ;;
 		esac
 		echo " $p"
@@ -1218,14 +1223,12 @@ install_pkg_deb ()
 		log_file="${SRC}/output/${LOG_SUBPATH}/install.log"
 	fi
 
-	echo -e "\nInstalling packages in function: $_function" "[$_file:$_line]" >>$log_file
+	apt-get -q update
-	echo -e "\nIncoming list:" >>$log_file
+	if [ "$?" != "0" ]; then echo "apt cannot update" >>$log_file; fi
-	printf "%-30s %-30s %-30s %-30s\n" $list >>$log_file
+	if $need_upgrade; then
-	echo "" >>$log_file
+		apt-get -y upgrade
-
+		if [ "$?" != "0" ]; then echo "apt cannot upgrade" >>$log_file; fi
-	(	apt-get -qq update
+	fi
-		if $need_upgrade; then apt-get -y upgrade;fi
-	) 2>>$log_file
 
 	# If the package is not installed, check the latest
 	# up-to-date version in the apt cache.
@@ -1242,18 +1245,30 @@ install_pkg_deb ()
 
 	# This information should be logged.
 	if [ -s $tmp_file ]; then
+		echo -e "\nInstalling packages in function: $_function" "[$_file:$_line]" \
+		>>$log_file
+		echo -e "\nIncoming list:" >>$log_file
+		printf "%-30s %-30s %-30s %-30s\n" $list >>$log_file
+		echo "" >>$log_file
 		cat $tmp_file >>$log_file
 	fi
 
 	if [ -n "$for_install" ]; then
 
-		apt-get install -qq -y --no-install-recommends $for_install 2>>$log_file
+		apt-get install -qq -y --no-install-recommends $for_install
+		echo -e "\nPackages installed:" >>$log_file
+		dpkg-query -W \
+		  -f '${binary:Package;-27} ${Version;-23}\n' \
+		  $for_install >>$log_file
 
-		# We will show the status after installation
+	fi
+
+	# We will show the status after installation all listed
+	if $need_verbose; then
 		echo -e "\nstatus after installation:" >>$log_file
 		dpkg-query -W \
 		  -f '${binary:Package;-27} ${Version;-23} [ ${Status} ]\n' \
-		  $for_install >>$log_file
+		  $list >>$log_file
 	fi
 
 	if $need_clean;then apt-get clean; fi
@@ -1312,17 +1327,6 @@ prepare_host()
 	else
 		local offline=false
 	fi
-# build aarch64
-  if [[ $(dpkg --print-architecture) != arm64 ]]; then
-
-	if [[ $(dpkg --print-architecture) != amd64 ]]; then
-		display_alert "Please read documentation to set up proper compilation environment"
-		display_alert "https://www.armbian.com/using-armbian-tools/"
-		exit_with_error "Running this tool on non x86_64 build host is not supported"
-	fi
-
-# build aarch64
-  fi
 
 	# wait until package manager finishes possible system maintanace
 	wait_for_package_manager
@@ -1338,31 +1342,34 @@ prepare_host()
 	# packages list for host
 	# NOTE: please sync any changes here with the Dockerfile and Vagrantfile
 
-# build aarch64
+	local hostdeps="acl aptly aria2 bc binfmt-support bison btrfs-progs       \
+	build-essential  ca-certificates ccache cpio cryptsetup curl              \
+	debian-archive-keyring debian-keyring debootstrap device-tree-compiler    \
+	dialog dirmngr dosfstools dwarves f2fs-tools fakeroot flex gawk           \
+	gcc-arm-linux-gnueabihf gdisk gnupg1 gpg imagemagick jq kmod libbison-dev \
+	libc6-dev-armhf-cross libelf-dev libfdt-dev libfile-fcntllock-perl        \
+	libfl-dev liblz4-tool libncurses-dev libpython2.7-dev libssl-dev          \
+	libusb-1.0-0-dev linux-base locales lzop ncurses-base ncurses-term        \
+	nfs-kernel-server ntpdate p7zip-full parted patchutils pigz pixz          \
+	pkg-config pv python3-dev python3-distutils qemu-user-static rsync swig   \
+	systemd-container u-boot-tools udev unzip uuid-dev wget whiptail zip      \
+	zlib1g-dev"
+
   if [[ $(dpkg --print-architecture) == amd64 ]]; then
 
-	local hostdeps="wget ca-certificates device-tree-compiler pv bc lzop zip binfmt-support build-essential ccache debootstrap ntpdate \
+	hostdeps+=" distcc lib32ncurses-dev lib32stdc++6 libc6-i386 zlib1g:i386"
-	gawk gcc-arm-linux-gnueabihf qemu-user-static u-boot-tools uuid-dev zlib1g-dev unzip libusb-1.0-0-dev fakeroot \
+	grep -q i386 <(dpkg --print-foreign-architectures) || dpkg --add-architecture i386
-	parted pkg-config libncurses5-dev whiptail debian-keyring debian-archive-keyring f2fs-tools libfile-fcntllock-perl rsync libssl-dev \
+
-	nfs-kernel-server btrfs-progs ncurses-term p7zip-full kmod dosfstools libc6-dev-armhf-cross imagemagick \
+  elif [[ $(dpkg --print-architecture) == arm64 ]]; then
-	curl patchutils liblz4-tool libpython2.7-dev linux-base swig aptly acl python3-dev python3-distutils \
+
-	locales ncurses-base pixz dialog systemd-container udev libfdt-dev libelf-dev lib32stdc++6 libc6-i386 lib32ncurses5 lib32tinfo5 \
+	hostdeps+=" gcc-arm-linux-gnueabi gcc-arm-none-eabi libc6 libc6-amd64-cross qemu"
-	bison libbison-dev flex libfl-dev cryptsetup gpg gnupg1 cpio aria2 pigz dirmngr python3-distutils jq distcc gdisk dwarves"
 
-# build aarch64
   else
 
-	local hostdeps="wget ca-certificates device-tree-compiler pv bc lzop zip binfmt-support build-essential ccache debootstrap ntpdate \
+	display_alert "Please read documentation to set up proper compilation environment"
-	gawk gcc-arm-linux-gnueabihf gcc-arm-linux-gnueabi gcc-arm-none-eabi \
+	display_alert "https://www.armbian.com/using-armbian-tools/"
-	qemu-user-static u-boot-tools uuid-dev zlib1g-dev unzip libusb-1.0-0-dev fakeroot \
+	exit_with_error "Running this tool on non x86_64 build host is not supported"
-	parted pkg-config libncurses5-dev whiptail debian-keyring debian-archive-keyring f2fs-tools libfile-fcntllock-perl rsync libssl-dev \
-	nfs-kernel-server btrfs-progs ncurses-term p7zip-full kmod dosfstools libc6-amd64-cross libc6-dev-armhf-cross imagemagick \
-	curl patchutils liblz4-tool libpython2.7-dev linux-base swig aptly acl python3-dev \
-	locales ncurses-base pixz dialog systemd-container udev libfdt-dev libelf-dev libc6 qemu \
-	bison libbison-dev flex libfl-dev cryptsetup gpg gnupg1 cpio aria2 pigz \
-	dirmngr python3-distutils jq gdisk dwarves"
 
-# build aarch64
   fi
 
 	# Add support for Ubuntu 20.04, 21.04 and Mint 20.x
@@ -1398,17 +1405,6 @@ prepare_host()
 		fi
 	fi
 
-# build aarch64
-  if [[ $(dpkg --print-architecture) == amd64 ]]; then
-
-	if [[ -z $HOSTRELEASE || $HOSTRELEASE =~ ^(focal|debbie|buster|bullseye|impish|hirsute|ulyana|ulyssa|uma)$ ]]; then
-	    hostdeps="${hostdeps/lib32ncurses5 lib32tinfo5/lib32ncurses6 lib32tinfo6}"
-	fi
-
-	grep -q i386 <(dpkg --print-foreign-architectures) || dpkg --add-architecture i386
-# build aarch64
-  fi
-
 	if systemd-detect-virt -q -c; then
 		display_alert "Running in container" "$(systemd-detect-virt)" "info"
 		# disable apt-cacher unless NO_APT_CACHER=no is not specified explicitly
@@ -1429,12 +1425,10 @@ prepare_host()
 	# Skip verification if you are working offline
 	if ! $offline; then
 
-	# warning: apt-cacher-ng will fail if installed and used both on host and in container/chroot environment with shared network
+	# warning: apt-cacher-ng will fail if installed and used both on host and in
+	# container/chroot environment with shared network
 	# set NO_APT_CACHER=yes to prevent installation errors in such case
-	if [[ $NO_APT_CACHER != yes ]]; then hostdeps="$hostdeps apt-cacher-ng"; fi
+	if [[ $NO_APT_CACHER != yes ]]; then hostdeps+=" apt-cacher-ng"; fi
-
-	local deps=()
-	local installed=$(dpkg-query -W -f '${db:Status-Abbrev}|${binary:Package}\n' '*' 2>/dev/null | grep '^ii' | awk -F '|' '{print $2}' | cut -d ':' -f 1)
 
 	export EXTRA_BUILD_DEPS=""
 	call_extension_method "add_host_dependencies" <<- 'ADD_HOST_DEPENDENCIES'
@@ -1442,9 +1436,7 @@ prepare_host()
 	you can add packages to install, space separated, to ${EXTRA_BUILD_DEPS} here.
 	ADD_HOST_DEPENDENCIES
 
-	for packet in $hostdeps ${EXTRA_BUILD_DEPS}; do
+	if [ -n "${EXTRA_BUILD_DEPS}" ]; then hostdeps+=" ${EXTRA_BUILD_DEPS}"; fi
-		if ! grep -q -x -e "$packet" <<< "$installed"; then deps+=("$packet"); fi
-	done
 
 	# distribution packages are buggy, download from author
 
@@ -1466,15 +1458,15 @@ prepare_host()
 # build aarch64
   fi
 
-	if [[ ${#deps[@]} -gt 0 ]]; then
+	display_alert "Installing build dependencies"
-		display_alert "Installing build dependencies"
+	# don't prompt for apt cacher selection
-		# don't prompt for apt cacher selection
+	sudo echo "apt-cacher-ng    apt-cacher-ng/tunnelenable      boolean false" | sudo debconf-set-selections
-		sudo echo "apt-cacher-ng    apt-cacher-ng/tunnelenable      boolean false" | sudo debconf-set-selections
+
-		apt-get -q update
+	LOG_OUTPUT_FILE="${DEST}"/${LOG_SUBPATH}/hostdeps.log
-		apt-get -y upgrade
+	install_pkg_deb "upgrade $hostdeps"
-		apt-get -q -y --no-install-recommends install -o Dpkg::Options::='--force-confold' "${deps[@]}" | tee -a "${DEST}"/${LOG_SUBPATH}/hostdeps.log
+	unset LOG_OUTPUT_FILE
-		update-ccache-symlinks
+
-	fi
+	update-ccache-symlinks
 
 	export FINAL_HOST_DEPS="$hostdeps ${EXTRA_BUILD_DEPS}"
 	call_extension_method "host_dependencies_ready" <<- 'HOST_DEPENDENCIES_READY'
@@ -1491,16 +1483,6 @@ prepare_host()
 		ntpdate -s "${NTP_SERVER:-pool.ntp.org}"
 	fi
 
-# build aarch64
-  if [[ $(dpkg --print-architecture) == amd64 ]]; then
-
-	if [[ $(dpkg-query -W -f='${db:Status-Abbrev}\n' 'zlib1g:i386' 2>/dev/null) != *ii* ]]; then
-		apt-get install -qq -y --no-install-recommends zlib1g:i386 >/dev/null 2>&1
-	fi
-
-# build aarch64
-  fi
-
 	# create directory structure
 	mkdir -p "${SRC}"/{cache,output} "${USERPATCHES_PATH}"
 	if [[ -n $SUDO_USER ]]; then
@@ -1562,7 +1544,9 @@ prepare_host()
 		else
 			display_alert "Ignoring toolchains" "SKIP_EXTERNAL_TOOLCHAINS: ${SKIP_EXTERNAL_TOOLCHAINS}" "info"
 		fi
-	fi # check offline
+	fi
+
+  fi # check offline
 
 	# enable arm binary format so that the cross-architecture chroot environment will work
 	if [[ $KERNEL_ONLY != yes ]]; then
@@ -1574,9 +1558,6 @@ prepare_host()
 		fi
 	fi
 
-# build aarch64
-  fi
-
 	[[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && cp "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh
 
 	if [[ ! -f "${USERPATCHES_PATH}"/README ]]; then