armbian-next: cleanup, kill and add to-do's, remove dead code, turn down logging, squash future shortcircuit bugs

- killed `[[ $ROOTFS_TYPE != ext4 ]] && display_alert "Assuming ${BOARD} ${BRANCH} kernel supports ${ROOTFS_TYPE}" "" "wrn"`
  - which definitely didn't belong in rootfs
- disable usage of run_host_command_logged_long_running for kernel-make
- stop lying about long_running stuff being any different from non-long_running versions
- stop lying about 'set -e' when there's still a bunch of pipes for "pv" and stuff all around

lib/functions/compilation/kernel-make.sh

@@ -62,7 +62,7 @@ function run_kernel_make_dialog() {
 
 function run_kernel_make_long_running() {
 	local seconds_start=${SECONDS} # Bash has a builtin SECONDS that is seconds since start of script
-	KERNEL_MAKE_RUNNER="run_host_command_logged_long_running" KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
+	KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
 	display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug"
 }
 

lib/functions/compilation/kernel.sh

@@ -187,7 +187,7 @@ function kernel_build() {
 	display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info"
 	# make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \ # @TODO this will be summarised in the log file eventually, but shown in realtime in screen
 	do_with_ccache_statistics \
-		run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}"
+		run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}" # "V=1" # "-s" silent mode, "V=1" verbose mode
 
 	display_alert "Kernel built in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
 }

lib/functions/configuration/aggregation.sh

@@ -37,8 +37,8 @@ function aggregate_all_packages_python() {
 
 		"ARCH=${ARCH}"
 		"RELEASE=${RELEASE}"
-		"LINUXFAMILY=${LINUXFAMILY}"
+		"LINUXFAMILY=" # empty. Important: LINUXFAMILY is no longer considered for package aggregation.
-		"BOARD=${BOARD}"
+		"BOARD="       # empty. Important: BOARD is no longer considered for package aggregation
 		"USERPATCHES_PATH=${USERPATCHES_PATH}"
 		"SELECTED_CONFIGURATION=${SELECTED_CONFIGURATION}"
 

lib/functions/configuration/main-config.sh

@@ -110,6 +110,7 @@ function do_main_configuration() {
 			exit_with_error "Unknown rootfs type: ROOTFS_TYPE='${ROOTFS_TYPE}'"
 			;;
 	esac
+	
 
 	# Support for LUKS / cryptroot
 	if [[ $CRYPTROOT_ENABLE == yes ]]; then
@@ -442,4 +443,6 @@ function source_family_config_and_arch() {
 	display_alert "Sourcing arch configuration" "${ARCH}.conf" "info"
 	# shellcheck source=/dev/null
 	source "${SRC}/config/sources/${ARCH}.conf"
+	
+	return 0
 }

lib/functions/host/prepare-host.sh

@@ -50,29 +50,6 @@ function prepare_host_noninteractive() {
 	export LC_ALL="en_US.UTF-8"
 	export LC_MESSAGES="en_US.UTF-8"
 
-	# @TODO kill this
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## # armbian-next: Armbian mirrors and the download code is highly unstable; disable by default
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## # armbian-next: set `SKIP_ARMBIAN_ROOTFS_CACHE=no` to enable
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## # don't use mirrors that throws garbage on 404
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## if [[ -z ${ARMBIAN_MIRROR} && "${SKIP_ARMBIAN_REPO}" != "yes" && "${SKIP_ARMBIAN_ROOTFS_CACHE:-"yes"}" != "yes" ]]; then
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 	display_alert "Determining best Armbian mirror to use" "via redirector" "debug"
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 	declare -i armbian_mirror_tries=1
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 	while true; do
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		display_alert "Obtaining Armbian mirror" "via https://redirect.armbian.com" "debug"
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		ARMBIAN_MIRROR=$(wget -SO- -T 1 -t 1 https://redirect.armbian.com 2>&1 | egrep -i "Location" | awk '{print $2}' | head -1)
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		if [[ ${ARMBIAN_MIRROR} != *armbian.hosthatch* ]]; then
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 			display_alert "Obtained Armbian mirror OK" "${ARMBIAN_MIRROR}" "debug"
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 			break
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		else
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 			display_alert "Obtained Armbian mirror is invalid, retrying..." "${ARMBIAN_MIRROR}" "debug"
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		fi
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		armbian_mirror_tries=$((armbian_mirror_tries + 1))
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		if [[ $armbian_mirror_tries -ge 5 ]]; then
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 			exit_with_error "Unable to obtain ARMBIAN_MIRROR after ${armbian_mirror_tries} tries. Please set ARMBIAN_MIRROR to a valid mirror manually, or avoid the automatic mirror selection by setting SKIP_ARMBIAN_REPO=yes"
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 		fi
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## 	done
-	## DEAD CODE? nothing uses ARMBIAN_MIRROR ## fi
-
 	declare -g USE_LOCAL_APT_DEB_CACHE=${USE_LOCAL_APT_DEB_CACHE:-yes} # Use SRC/cache/aptcache as local apt cache by default
 	display_alert "Using local apt cache?" "USE_LOCAL_APT_DEB_CACHE: ${USE_LOCAL_APT_DEB_CACHE}" "debug"
 

lib/functions/logging/runners.sh

@@ -21,12 +21,6 @@ function chroot_sdcard_apt_get_install_dry_run() {
 }
 
 function chroot_sdcard_apt_get_update() {
-	# --list-cleanup
-	#     This option is on by default; use --no-list-cleanup to turn it off. When it is on, apt-get will
-	#     automatically manage the contents of /var/lib/apt/lists to ensure that obsolete files are erased. The only
-	#     reason to turn it off is if you frequently change your sources list. Configuration Item:
-	#     APT::Get::List-Cleanup.
-
 	apt_logging="-q" chroot_sdcard_apt_get update
 }
 
@@ -54,8 +48,14 @@ function chroot_sdcard_apt_get() {
 		display_alert "Not using apt-cacher-ng, nor proxy" "no proxy/acng" "debug"
 	fi
 
-	apt_params+=(-o "Dpkg::Use-Pty=0")          # Please be quiet
+	apt_params+=(-o "Dpkg::Use-Pty=0") # Please be quiet
-	apt_params+=(-o "APT::Get::List-Cleanup=0") # Don't cleanup package lists. We do it ourselves when needed.
+
+	# --list-cleanup
+	#     This option is on by default; use --no-list-cleanup to turn it off. When it is on, apt-get will
+	#     automatically manage the contents of /var/lib/apt/lists to ensure that obsolete files are erased. The only
+	#     reason to turn it off is if you frequently change your sources list. Configuration Item:
+	#     APT::Get::List-Cleanup.
+	apt_params+=(-o "APT::Get::List-Cleanup=0") # Armbian frequently changes ours sources list; it's dynamic via aggregation
 
 	if [[ "${DONT_MAINTAIN_APT_CACHE:-no}" == "yes" ]]; then
 		# Configure Clean-Installed to off
@@ -111,21 +111,9 @@ function chroot_sdcard_with_stdout() {
 	TMPDIR="" chroot "${SDCARD}" "$@"
 }
 
-function chroot_custom_long_running() {
+function chroot_custom_long_running() { # any pipe causes the left-hand side to subshell and caos ensues. it's just like chroot_custom()
 	local target=$1
 	shift
-
-	# @TODO: disabled, the pipe causes the left-hand side to subshell and caos ensues.
-	# local _exit_code=1
-	# if [[ "${SHOW_LOG}" == "yes" ]] || [[ "${CI}" == "true" ]]; then
-	# 	TMPDIR="" run_host_command_logged_raw chroot "${target}" /bin/bash -e -o pipefail -c "$*"
-	# 	_exit_code=$?
-	# else
-	# 	TMPDIR="" run_host_command_logged_raw chroot "${target}" /bin/bash -e -o pipefail -c "$*" | pv -N "$(logging_echo_prefix_for_pv "${INDICATOR:-compile}")" --progress --timer --line-mode --force --cursor --delay-start 0 -i "0.5"
-	# 	_exit_code=$?
-	# fi
-	# return $_exit_code
-
 	raw_command="$*" raw_extra="chroot_custom_long_running" TMPDIR="" run_host_command_logged_raw chroot "${target}" /bin/bash -e -o pipefail -c "$*"
 }
 
@@ -135,26 +123,6 @@ function chroot_custom() {
 	raw_command="$*" raw_extra="chroot_custom" TMPDIR="" run_host_command_logged_raw chroot "${target}" /bin/bash -e -o pipefail -c "$*"
 }
 
-# for long-running, host-side expanded bash invocations.
-# the user gets a pv-based spinner based on the number of lines that flows to stdout (log messages).
-# the raw version is already redirect stderr to stdout, and we'll be running under do_with_logging,
-# so: _the stdout must flow_!!!
-function run_host_command_logged_long_running() {
-	# @TODO: disabled. The Pipe used for "pv" causes the left-hand side to run in a subshell.
-	#local _exit_code=1
-	#if [[ "${SHOW_LOG}" == "yes" ]] || [[ "${CI}" == "true" ]]; then
-	#	run_host_command_logged_raw /bin/bash -e -o pipefail-c "$*"
-	#	_exit_code=$?
-	#else
-	#	run_host_command_logged_raw /bin/bash -e -o pipefail -c "$*" | pv -N "$(logging_echo_prefix_for_pv "${INDICATOR:-compile}")  " --progress --timer --line-mode --force --cursor --delay-start 0 -i "2"
-	#	_exit_code=$?
-	#fi
-	#return $_exit_code
-
-	# Run simple and exit with it's code. Sorry.
-	raw_command="$*" run_host_command_logged_raw /bin/bash -e -o pipefail -c "$*"
-}
-
 # For installing packages host-side. Not chroot!
 function host_apt_get_install() {
 	host_apt_get --no-install-recommends install "$@"
@@ -192,6 +160,11 @@ function run_host_x86_binary_logged() {
 	run_host_command_logged "${qemu_invocation[@]}" # Exit with this result code
 }
 
+# Run simple and exit with it's code. Exactly the same as run_host_command_logged(). Used to have pv pipe, but that causes chaos.
+function run_host_command_logged_long_running() {
+	raw_command="${raw_command:-"$*"}" run_host_command_logged_raw /bin/bash -e -o pipefail -c "$*"
+}
+
 # run_host_command_logged is the very basic, should be used for everything, but, please use helpers above, this is very low-level.
 function run_host_command_logged() {
 	raw_command="${raw_command:-"$*"}" run_host_command_logged_raw /bin/bash -e -o pipefail -c "$*"

lib/functions/main/build-packages.sh

@@ -120,14 +120,6 @@ function main_default_build_packages() {
 		LOG_SECTION="create_bsp_desktop_package" do_with_logging create_bsp_desktop_package
 	fi
 
-	# @TODO: dead code, remove
-	## DEAD CODE IN ARMBIAN-NEXT ## # build additional packages
-	## DEAD CODE IN ARMBIAN-NEXT ## if [[ $EXTERNAL_NEW == compile ]]; then
-	## DEAD CODE IN ARMBIAN-NEXT ## 	display_alert "Building external packages" "$EXTERNAL - WARNING: this has not been armbian-nextifed yet and will fail." "warn"
-	## DEAD CODE IN ARMBIAN-NEXT ## 	exit_with_error "buildpkg chroot_build_packages() nas not been ported to armbian-next yet, rpardini is a bum"
-	## DEAD CODE IN ARMBIAN-NEXT ## 	LOG_SECTION="chroot_build_packages" do_with_logging chroot_build_packages
-	## DEAD CODE IN ARMBIAN-NEXT ## fi
-
 	# Reset owner of DEB_STORAGE, if needed. Might be a lot of packages there, but such is life.
 	# @TODO: might be needed also during 'cleanup': if some package fails, the previous package might be left owned by root.
 	reset_uid_owner "${DEB_STORAGE}"

lib/functions/main/config-prepare.sh

@@ -31,7 +31,7 @@ function prep_conf_main_build_single() {
 	display_alert "Configuration prepared for BOARD build" "${BOARD}.${BOARD_TYPE}" "info"
 }
 
-# Lean version, for building stuff that doesn't need BOARD/BOARDFAMILY; never interactive.
+# Lean version, for building rootfs, that doesn't need BOARD/BOARDFAMILY; never interactive.
 function prep_conf_main_only_rootfs() {
 	LOG_SECTION="config_early_init" do_with_conditional_logging config_early_init
 

lib/functions/main/rootfs-image.sh

@@ -2,7 +2,6 @@
 
 function build_rootfs_and_image() {
 	display_alert "Checking for rootfs cache" "$(echo "${BRANCH} ${BOARD} ${RELEASE} ${DESKTOP_APPGROUPS_SELECTED} ${DESKTOP_ENVIRONMENT} ${BUILD_MINIMAL}" | tr -s " ")" "info"
-	[[ $ROOTFS_TYPE != ext4 ]] && display_alert "Assuming ${BOARD} ${BRANCH} kernel supports ${ROOTFS_TYPE}" "" "wrn"
 
 	LOG_SECTION="prepare_rootfs_build_params_and_trap" do_with_logging prepare_rootfs_build_params_and_trap
 

lib/functions/rootfs/create-cache.sh

@@ -36,11 +36,15 @@ function calculate_rootfs_cache_id() {
 	[[ ${BUILD_MINIMAL} == yes ]] && cache_type="minimal"
 	declare -g -r cache_type="${cache_type}"
 
-	display_alert "calculate_rootfs_cache_id: done with packages-hash" "${packages_hash}" "warn"
+	display_alert "calculate_rootfs_cache_id: done with packages-hash" "${packages_hash}" "debug"
 }
 
 # this gets from cache or produces a basic new rootfs, ready, but not mounted, at "$SDCARD"
 function get_or_create_rootfs_cache_chroot_sdcard() {
+	if [[ "${ROOT_FS_CREATE_ONLY}" == yes ]]; then
+		exit_with_error "Using deprecated ROOT_FS_CREATE_ONLY=yes, that is not longer supported. use 'rootfs' CLI command."
+	fi
+
 	# validate "${SDCARD}" is set. it does not exist, yet...
 	if [[ -z "${SDCARD}" ]]; then
 		exit_with_error "SDCARD is not set at get_or_create_rootfs_cache_chroot_sdcard()"
@@ -71,7 +75,7 @@ function get_or_create_rootfs_cache_chroot_sdcard() {
 	# Make ROOTFSCACHE_VERSION global at this point, in case it was not.
 	declare -g ROOTFSCACHE_VERSION="${ROOTFSCACHE_VERSION}"
 
-	display_alert "ROOTFSCACHE_VERSION found online or preset" "${ROOTFSCACHE_VERSION}" "warn"
+	display_alert "ROOTFSCACHE_VERSION found online or preset" "${ROOTFSCACHE_VERSION}" "debug"
 
 	calculate_rootfs_cache_id # this sets packages_hash and cache_type
 
@@ -80,9 +84,9 @@ function get_or_create_rootfs_cache_chroot_sdcard() {
 	get_rootfs_cache_list_into_array_variable # sets cache_list
 
 	# Show the number of items in the cache_list array
-	display_alert "Found possible rootfs caches: " "${#cache_list[@]}" "warn"
+	display_alert "Found possible rootfs caches: " "${#cache_list[@]}" "debug"
 
-	display_alert "ROOTFSCACHE_VERSION after getting cache list" "${ROOTFSCACHE_VERSION}" "warn"
+	display_alert "ROOTFSCACHE_VERSION after getting cache list" "${ROOTFSCACHE_VERSION}" "debug"
 
 	declare possible_cached_version
 	for possible_cached_version in "${cache_list[@]}"; do
@@ -90,11 +94,6 @@ function get_or_create_rootfs_cache_chroot_sdcard() {
 		local cache_name="${ARCH}-${RELEASE}-${cache_type}-${packages_hash}-${ROOTFSCACHE_VERSION}.tar.zst"
 		local cache_fname="${SRC}/cache/rootfs/${cache_name}"
 
-		if [[ "$ROOT_FS_CREATE_ONLY" == yes ]]; then
-			display_alert "Using deprecated" "ROOT_FS_CREATE_ONLY=yes during search for existing cache" "warn"
-			break
-		fi
-
 		display_alert "Checking cache" "$cache_name" "info"
 
 		# if aria2 file exists download didn't succeeded
@@ -111,22 +110,27 @@ function get_or_create_rootfs_cache_chroot_sdcard() {
 		fi
 	done
 
-	display_alert "ROOTFSCACHE_VERSION after looping" "${ROOTFSCACHE_VERSION}" "warn"
+	display_alert "ROOTFSCACHE_VERSION after looping" "${ROOTFSCACHE_VERSION}" "debug"
 
-	# if not "only" creating rootfs and cache exists, extract it
+	# if cache found, extract it
 	# if aria2 file exists, download didn't succeeded, so skip it
-	# @TODO this could be named IGNORE_EXISTING_ROOTFS_CACHE=yes
+	# we can ignore existing cache with IGNORE_EXISTING_ROOTFS_CACHE=yes
-	if [[ "${ROOT_FS_CREATE_ONLY}" != "yes" && -f "${cache_fname}" && ! -f "${cache_fname}.aria2" ]]; then
+	if [[ "${IGNORE_EXISTING_ROOTFS_CACHE}" != "yes" && -f "${cache_fname}" && ! -f "${cache_fname}.aria2" ]]; then
 		# validate sanity
 		[[ "x${SDCARD}x" == "xx" ]] && exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: SDCARD: ${SDCARD} is not set"
 
 		local date_diff=$((($(date +%s) - $(stat -c %Y "${cache_fname}")) / 86400))
 		display_alert "Extracting $cache_name" "$date_diff days old" "info"
 		pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "extract_rootfs") $cache_name" "$cache_fname" | zstdmt -dc | tar xp --xattrs -C "${SDCARD}"/
-		# @TODO: this never runs, since 'set -e' ("errexit") is in effect, and https://github.com/koalaman/shellcheck/wiki/SC2181
-		# [[ $? -ne 0 ]] && rm $cache_fname && exit_with_error "Cache $cache_fname is corrupted and was deleted. Restart."
 
-		#echo >&2 # newline to stderr after using pv?
+		declare -a pv_tar_zstdmt_pipe_status=("${PIPESTATUS[@]}") # capture and the pipe_status array from PIPESTATUS
+		declare one_pipe_status
+		for one_pipe_status in "${pv_tar_zstdmt_pipe_status[@]}"; do
+			if [[ "$one_pipe_status" != "0" ]]; then
+				exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: ${cache_fname} failed (${pv_tar_zstdmt_pipe_status[*]}) - corrupt cache?"
+			fi
+		done
+
 		wait_for_disk_sync "after restoring rootfs cache"
 
 		run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf
@@ -138,13 +142,6 @@ function get_or_create_rootfs_cache_chroot_sdcard() {
 		create_new_rootfs_cache
 	fi
 
-	# @TODO: remove after killing usages
-	#  used for internal purposes. Faster rootfs cache rebuilding
-	if [[ "${ROOT_FS_CREATE_ONLY}" == "yes" ]]; then
-		display_alert "Using, does nothing" "ROOT_FS_CREATE_ONLY=yes, late in get_or_create_rootfs_cache_chroot_sdcard" "warning"
-		# this used to try to disable traps, umount and exit. no longer. let the function finish
-	fi
-
 	return 0
 }
 
@@ -167,7 +164,7 @@ function create_new_rootfs_cache() {
 
 	# needed for backend to keep current only @TODO: still needed?
 	echo "$cache_fname" > "${cache_fname}.current"
-	
+
 	# define a readonly global with the name of the cache
 	declare -g -r BUILT_ROOTFS_CACHE_NAME="${cache_name}"
 	declare -g -r BUILT_ROOTFS_CACHE_FILE="${cache_fname}"
@@ -200,10 +197,10 @@ function get_rootfs_cache_list_into_array_variable() {
 	} | sort | uniq | sort -r)"
 
 	# Show the contents
-	display_alert "Available cache versions number" "${#local_cache_list[*]}" "warn"
+	display_alert "Available cache versions number" "${#local_cache_list[*]}" "debug"
 	# Loop each and show
 	for cache_version in "${local_cache_list[@]}"; do
-		display_alert "One available cache version" "${cache_version}" "warn"
+		display_alert "One available cache version" "${cache_version}" "debug"
 	done
 
 	# return the list to outer scope

lib/functions/rootfs/trap-rootfs.sh

@@ -5,9 +5,9 @@ function prepare_rootfs_build_params_and_trap() {
 
 	# stage: clean and create directories
 	run_host_command_logged rm -rfv "${SDCARD}" "${MOUNT}"
-	run_host_command_logged mkdir -pv "${SDCARD}" "${MOUNT}" "${SRC}/cache/rootfs" "${DEST}/images" # @TODO images shouldn't be here up
+	run_host_command_logged mkdir -pv "${SDCARD}" "${MOUNT}" "${SRC}/cache/rootfs" "${DEST}/images" # @TODO images needs its own trap
 
-	# bind mount rootfs if defined # @TODO: is this used? it is never unmounted
+	# bind mount rootfs if defined # @TODO: is this used? Igor's NAS?
 	if [[ -d "${ARMBIAN_CACHE_ROOTFS_PATH}" ]]; then
 		display_alert "Warning, using untested code path" "ARMBIAN_CACHE_ROOTFS_PATH" "warn"
 		mountpoint -q "${SRC}"/cache/rootfs && umount "${SRC}"/cache/rootfs