armbian-next: kernel: deterministic .config mtime handling; kernel build logging sections reorg

- .config is now compared with previous one byte-wise, diff shown, and mtime preserved directly via `cp -p`
- this does away with the previous mtime-based BS I had written during the fasthash era (and thus remove file-mtime.sh, regen lib)
- split some functions, to better control the interactive piece in the _middle_ of kernel config process
- no more `kernel_config_maybe_interactive()`, now `kernel_config()` that controls its own logging sections
- extract `kernel_determine_toolchain()` out of config into make
- introduce internal hook `armbian_kernel_config()` that runs before custom version (for .config defaults, coming soon)

extensions/kernel-localmodconfig.sh

@@ -13,10 +13,9 @@ function extension_prepare_config__prepare_localmodconfig() {
 
 # This needs much more love than this. can be used to make "light" versions of kernels, that compile 3x-5x faster or more
 function custom_kernel_config_post_defconfig__apply_localmodconfig() {
-	display_alert "localmodconfig with lsmod" "${KERNEL_CONFIG_FROM_LSMOD}" "warn"
 	if [[ "a${KERNEL_CONFIG_FROM_LSMOD}a" != "aa" ]]; then
+		display_alert "localmodconfig with lsmod" "${KERNEL_CONFIG_FROM_LSMOD}" "warn"
 		local lsmod_file="${SRC}/userpatches/lsmod/${KERNEL_CONFIG_FROM_LSMOD}.lsmod"
-		run_kernel_make "LSMOD=${lsmod_file}" localmodconfig
+		run_kernel_make "LSMOD=${lsmod_file}" localmodconfig "> /dev/null" # hide output, it's way too long. stderr still shows
-		kernel_config_mtime=$(get_file_modification_time ".config") # capture the mtime of the config file after the localmodconfig
 	fi
 }

lib/functions/compilation/kernel-config.sh

@@ -1,98 +1,85 @@
-function kernel_config_maybe_interactive() {
+function kernel_config() {
-	# Check if we're gonna do some interactive configuration; if so, don't run kernel_config under logging manager.
+	# check $kernel_work_dir is set and exists, or bail
-	if [[ $KERNEL_CONFIGURE != yes ]]; then
+	[[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set"
-		LOG_SECTION="kernel_config" do_with_logging do_with_hooks kernel_config
+	[[ ! -d "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir does not exist: ${kernel_work_dir}"
-	else
+	declare previous_config_filename=".config.armbian.previous"
-		LOG_SECTION="kernel_config_interactive" do_with_hooks kernel_config
+	declare kernel_config_source_filename="" # which actual .config was used?
+
+	LOG_SECTION="kernel_config_initialize" do_with_logging do_with_hooks kernel_config_initialize
+
+	if [[ "${KERNEL_CONFIGURE}" == "yes" ]]; then
+		# This piece is interactive, no logging
+		display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug"
+		run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}"
+
+		# Export, but log about it too.
+		LOG_SECTION="kernel_config_export" do_with_logging do_with_hooks kernel_config_export
 	fi
+
+	LOG_SECTION="kernel_config_finalize" do_with_logging do_with_hooks kernel_config_finalize
 }
 
-function kernel_config() {
+function kernel_config_initialize() {
-	# re-read kernel version after patching
+	display_alert "Configuring kernel" "${LINUXCONFIG}" "info"
-	version=$(grab_version "$kernel_work_dir")
 
-	display_alert "Compiling $BRANCH kernel" "$version" "info"
+	# If a `.config` already exists (from previous build), store it, preserving date.
-
+	# We will compare the result of the new configuration to it, and if the contents are the same, we'll restore the original date.
-	# compare with the architecture of the current Debian node
+	# This way we avoid unnecessary recompilation of the kernel; even if the .config contents
-	# if it matches we use the system compiler
+	# have not changed, the date will be different, and Kbuild will at least re-link everything.
-	if dpkg-architecture -e "${ARCH}"; then
+	if [[ -f "${kernel_work_dir}/.config" ]]; then
-		display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
+		display_alert "Preserving previous kernel configuration" "${previous_config_filename}" "debug"
-	else
+		run_host_command_logged cp -pv "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}"
-		display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
-		toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC")
-		[[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC"
 	fi
 
-	kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)"
+	# copy kernel config from configuration, userpatches
-	display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info"
+	if [[ "${KERNEL_KEEP_CONFIG}" == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then
-
+		display_alert "Using previously-exported kernel config" "${DEST}/config/$LINUXCONFIG.config" "info"
-	# copy kernel config
-	local COPY_CONFIG_BACK_TO=""
-
-	if [[ $KERNEL_KEEP_CONFIG == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then
-		display_alert "Using previous kernel config" "${DEST}/config/$LINUXCONFIG.config" "info"
 		run_host_command_logged cp -pv "${DEST}/config/${LINUXCONFIG}.config" .config
 	else
-		# @TODO: rpardini: this is too contrived, make obvious, use an array and a loop and stop repeating itself
+		# @TODO: rpardini: this is too contrived
 		if [[ -f $USERPATCHES_PATH/$LINUXCONFIG.config ]]; then
 			display_alert "Using kernel config provided by user" "userpatches/$LINUXCONFIG.config" "info"
 			run_host_command_logged cp -pv "${USERPATCHES_PATH}/${LINUXCONFIG}.config" .config
-			COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/${LINUXCONFIG}.config"
+			kernel_config_source_filename="${USERPATCHES_PATH}/${LINUXCONFIG}.config"
 		elif [[ -f "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" ]]; then
 			display_alert "Using kernel config provided by user in config/kernel folder" "config/kernel/${LINUXCONFIG}.config" "info"
 			run_host_command_logged cp -pv "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" .config
-			COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config"
+			kernel_config_source_filename="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config"
 		else
 			display_alert "Using kernel config file" "config/kernel/$LINUXCONFIG.config" "info"
 			run_host_command_logged cp -pv "${SRC}/config/kernel/${LINUXCONFIG}.config" .config
-			COPY_CONFIG_BACK_TO="${SRC}/config/kernel/${LINUXCONFIG}.config"
+			kernel_config_source_filename="${SRC}/config/kernel/${LINUXCONFIG}.config"
 		fi
 	fi
 
-	# Store the .config modification date at this time, for restoring later. Otherwise rebuilds.
+	# Start by running olddefconfig -- always.
-	local kernel_config_mtime
+	# It "updates" the config, using defaults from Kbuild files in the source tree.
-	kernel_config_mtime=$(get_file_modification_time ".config")
+	# It is worthy noting that on the first run, it builds the tools, so the host-side compiler has to be working,
+	# regardless of the cross-build toolchain.
+	run_kernel_make olddefconfig
 
+	# Run the core-armbian config modifications here, built-in extensions:
+	# 1) Enable the options for the extrawifi/drivers; so we don't need to worry about them when updating configs
+	# 2) Disable: debug, version shenanigans, signing, and compression of modules, to ensure sanity
+	call_extension_method "armbian_kernel_config" <<- 'ARMBIAN_KERNEL_CONFIG'
+		*Armbian-core default hook point for pre-olddefconfig Kernel config modifications*
+		NOT for user consumption. Do NOT use this hook, this is internal to Armbian.
+		Instead, use `custom_kernel_config` which runs later and can undo anything done by this step.
+	ARMBIAN_KERNEL_CONFIG
+
+	# Custom hooks receive a clean / updated config; depending on their modifications, they may need to run olddefconfig again.
 	call_extension_method "custom_kernel_config" <<- 'CUSTOM_KERNEL_CONFIG'
 		*Kernel .config is in place, still clean from git version*
 		Called after ${LINUXCONFIG}.config is put in place (.config).
-		Before any olddefconfig any Kconfig make is called.
 		A good place to customize the .config directly.
+		Armbian default Kconfig modifications have already been applied and can be overriden.
 	CUSTOM_KERNEL_CONFIG
 
 	display_alert "Kernel configuration" "${LINUXCONFIG}" "info"
 
-	if [[ $KERNEL_CONFIGURE != yes ]]; then
+}
-		run_kernel_make olddefconfig # @TODO: what is this? does it fuck up dates?
-	else
-		display_alert "Starting (non-interactive) kernel olddefconfig" "${LINUXCONFIG}" "debug"
-
-		run_kernel_make olddefconfig
-
-		# No logging for this. this is UI piece
-		display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug"
-		run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}"
-
-		# Capture new date. Otherwise changes not detected by make.
-		kernel_config_mtime=$(get_file_modification_time ".config")
-
-		# store kernel config in easily reachable place
-		mkdir -p "${DEST}"/config
-		display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info"
-		run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config"
-
-		# store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done.
-		if [[ "${COPY_CONFIG_BACK_TO}" != "" ]]; then
-			display_alert "Exporting new kernel config - git commit pending" "${COPY_CONFIG_BACK_TO}" "info"
-			run_host_command_logged cp -pv .config "${COPY_CONFIG_BACK_TO}"
-
-			# export defconfig
-			run_kernel_make savedefconfig
-			run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig"
-			run_host_command_logged cp -pv defconfig "${COPY_CONFIG_BACK_TO}.defconfig"
-		fi
-	fi
 
+function kernel_config_finalize() {
 	call_extension_method "custom_kernel_config_post_defconfig" <<- 'CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG'
 		*Kernel .config is in place, already processed by Armbian*
 		Called after ${LINUXCONFIG}.config is put in place (.config).
@@ -100,6 +87,38 @@ function kernel_config() {
 		A good place to customize the .config last-minute.
 	CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG
 
-	# Restore the date of .config. Above delta is a pure function, theoretically.
+	# Now, compare the .config with the previous one, and if they are the same, restore the original date.
-	set_files_modification_time "${kernel_config_mtime}" ".config"
+	# This way we avoid unnecessary recompilation of the kernel; even if the .config contents
+	# have not changed, the date will be different, and Kbuild will at least re-link everything.
+	if [[ -f "${kernel_work_dir}/${previous_config_filename}" ]]; then
+		# from "man cmp": Exit status is 0 if inputs are the same, 1 if different, 2 if trouble.
+		if cmp "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}"; then
+			display_alert "Kernel configuration unchanged from previous run" "optimizing for fast rebuilds" "cachehit"
+			run_host_command_logged cp -pv "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config"
+		else
+			display_alert "Kernel configuration changed from previous build" "optimizing for correctness" "info"
+			# ad: added lines; de: deleted lines; hd: header lines; ln: line numbers
+			# run_host_command_logged diff -u --color=always "--palette='rs=0:hd=1:ad=33:de=37:ln=36'" "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config" "|| true" # no errors please
+		fi
+		# either way, remove the previous file
+		run_host_command_logged rm -f "${kernel_work_dir}/${previous_config_filename}"
+	fi
+}
+
+function kernel_config_export() {
+	# store kernel config in easily reachable place
+	mkdir -p "${DEST}"/config
+	display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info"
+	run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config"
+
+	# store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done.
+	if [[ "${kernel_config_source_filename}" != "" ]]; then
+		display_alert "Exporting new kernel config - git commit pending" "${kernel_config_source_filename}" "info"
+		run_host_command_logged cp -pv .config "${kernel_config_source_filename}"
+
+		# export defconfig
+		run_kernel_make savedefconfig
+		run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig"
+		run_host_command_logged cp -pv defconfig "${kernel_config_source_filename}.defconfig"
+	fi
 }

lib/functions/compilation/kernel-make.sh

@@ -65,3 +65,17 @@ function run_kernel_make_long_running() {
 	display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug"
 }
 
+function kernel_determine_toolchain() {
+	# compare with the architecture of the current Debian node
+	# if it matches we use the system compiler
+	if dpkg-architecture -e "${ARCH}"; then
+		display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
+	else
+		display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
+		toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC")
+		[[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC"
+	fi
+
+	kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)"
+	display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info"
+}

lib/functions/compilation/kernel.sh

@@ -26,8 +26,8 @@ function compile_kernel() {
 	LOG_SECTION="kernel_maybe_clean" do_with_logging do_with_hooks kernel_maybe_clean
 
 	# Patching.
-	local version hash pre_patch_version
+	declare hash pre_patch_version
-	kernel_main_patching
+	kernel_main_patching # has it's own logging sections inside
 
 	# Stop after patching;
 	if [[ "${PATCH_ONLY}" == yes ]]; then
@@ -35,23 +35,30 @@ function compile_kernel() {
 		return 0
 	fi
 
-	local toolchain
+	# re-read kernel version after patching
-	kernel_config_maybe_interactive
+	declare version
+	version=$(grab_version "$kernel_work_dir")
+	display_alert "Compiling $BRANCH kernel" "$version" "info"
+
+	# determine the toolchain
+	declare toolchain
+	LOG_SECTION="kernel_determine_toolchain" do_with_logging do_with_hooks kernel_determine_toolchain
+
+	kernel_config # has it's own logging sections inside
 
 	# package the kernel-source .deb
 	LOG_SECTION="kernel_package_source" do_with_logging do_with_hooks kernel_package_source
 
 	# build via make and package .debs; they're separate sub-steps
-	LOG_SECTION="kernel_build_and_package" do_with_logging do_with_hooks kernel_build_and_package
+	kernel_prepare_build_and_package # has it's own logging sections inside
 
 	display_alert "Done with" "kernel compile" "debug"
-	cd "${kernel_work_dir}/.." || exit
 
-	rm -f linux-firmware-image-*.deb # remove firmware image packages here - easier than patching ~40 packaging scripts at once
+	LOG_SECTION="kernel_deploy_pkg" do_with_logging do_with_hooks kernel_deploy_pkg
-	run_host_command_logged rsync --remove-source-files -r ./*.deb "${DEB_STORAGE}/"
 
 	# kernel build worked; let's clean up the git-bundle cache, since the git-bare cache is proven working.
-	kernel_cleanup_bundle_artifacts
+	# @TODO: armbian-oleg: clean this earlier, so we save some 2gb on disk _during_ kernel compile, not after
+	LOG_SECTION="kernel_cleanup_bundle_artifacts" do_with_logging do_with_hooks kernel_cleanup_bundle_artifacts
 
 	return 0
 }
@@ -115,14 +122,14 @@ function kernel_package_source() {
 	display_alert "$(basename "${sources_pkg_dir}.deb" ".deb") packaged" "$((SECONDS - ts)) seconds, ${tarball_size} tarball, ${package_size} .deb" "info"
 }
 
-function kernel_build_and_package() {
+function kernel_prepare_build_and_package() {
-	local ts=${SECONDS}
+	declare -a build_targets
-
-	cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
-
-	local -a build_targets=("all") # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH}
 	declare kernel_dest_install_dir
-	kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.temp.install.target.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up.
+	declare -a install_make_params_quoted
+	declare -A kernel_install_dirs
+
+	build_targets=("all")                                              # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH}
+	kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up.
 
 	# define dict with vars passed and target directories
 	declare -A kernel_install_dirs=(
@@ -139,7 +146,6 @@ function kernel_build_and_package() {
 	fi
 
 	# loop over the keys above, get the value, create param value in array; also mkdir the dir
-	declare -a install_make_params_quoted
 	local dir_key
 	for dir_key in "${!kernel_install_dirs[@]}"; do
 		local dir="${kernel_install_dirs["${dir_key}"]}"
@@ -148,14 +154,38 @@ function kernel_build_and_package() {
 		install_make_params_quoted+=("${value}")
 	done
 
+	# Fire off the build & package
+	LOG_SECTION="kernel_build" do_with_logging do_with_hooks kernel_build
+
+	LOG_SECTION="kernel_package" do_with_logging do_with_hooks kernel_package
+}
+
+function kernel_build() {
+	local ts=${SECONDS}
+	cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
+
 	display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info"
 	make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \
 		do_with_ccache_statistics \
 		run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}"
 
+	display_alert "Kernel built in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
+}
+
+function kernel_package() {
+	local ts=${SECONDS}
 	cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
 	display_alert "Packaging kernel" "${LINUXFAMILY} ${LINUXCONFIG}" "info"
 	prepare_kernel_packaging_debs "${kernel_work_dir}" "${kernel_dest_install_dir}" "${version}" kernel_install_dirs
-
+	display_alert "Kernel packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
-	display_alert "Kernel built and packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
+}
+
+function kernel_deploy_pkg() {
+	cd "${kernel_work_dir}/.." || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
+
+	# @TODO: rpardini: this is kept for historical reasons... wth?
+	# remove firmware image packages here - easier than patching ~40 packaging scripts at once
+	run_host_command_logged rm -fv "linux-firmware-image-*.deb"
+
+	run_host_command_logged rsync -v --remove-source-files -r ./*.deb "${DEB_STORAGE}/"
 }

lib/functions/general/file-mtime.sh

@@ -1,38 +0,0 @@
-
-function get_file_modification_time() { # @TODO: This is almost always called from a subshell. No use throwing errors?
-	local -i file_date
-	if [[ ! -f "${1}" ]]; then
-		exit_with_error "Can't get modification time of nonexisting file" "${1}"
-		return 1
-	fi
-	# YYYYMMDDhhmm.ss - it is NOT a valid integer, but is what 'touch' wants for its "-t" parameter
-	# YYYYMMDDhhmmss - IS a valid integer and we can do math to it. 'touch' code will format it later
-	file_date=$(date +%Y%m%d%H%M%S -r "${1}")
-	display_alert "Read modification date for file" "${1} - ${file_date}" "timestamp"
-	echo -n "${file_date}"
-	return 0
-}
-
-function get_dir_modification_time() {
-	local -i file_date
-	if [[ ! -d "${1}" ]]; then
-		exit_with_error "Can't get modification time of nonexisting dir" "${1}"
-		return 1
-	fi
-	# YYYYMMDDhhmm.ss - it is NOT a valid integer, but is what 'touch' wants for its "-t" parameter
-	# YYYYMMDDhhmmss - IS a valid integer and we can do math to it. 'touch' code will format it later
-	file_date=$(date +%Y%m%d%H%M%S -r "${1}")
-	display_alert "Read modification date for DIRECTORY" "${1} - ${file_date}" "timestamp"
-	echo -n "${file_date}"
-	return 0
-}
-
-# This is for simple "set without thinking" usage, date preservation is done directly by process_patch_file
-function set_files_modification_time() {
-	local -i mtime="${1}"
-	local formatted_mtime
-	shift
-	display_alert "Setting date ${mtime}" "${*} (no newer check)" "timestamp"
-	formatted_mtime="${mtime:0:12}.${mtime:12}"
-	touch --no-create -m -t "${formatted_mtime}" "${@}"
-}

lib/library-functions.sh

@@ -442,15 +442,6 @@ set -o errexit  ## set -e : exit the script if any statement returns a non-true
 # shellcheck source=lib/functions/general/downloads.sh
 source "${SRC}"/lib/functions/general/downloads.sh
 
-# no errors tolerated. invoked before each sourced file to make sure.
-#set -o pipefail  # trace ERR through pipes - will be enabled "soon"
-#set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
-set -o errtrace # trace ERR through - enabled
-set -o errexit  ## set -e : exit the script if any statement returns a non-true return value - enabled
-### lib/functions/general/file-mtime.sh
-# shellcheck source=lib/functions/general/file-mtime.sh
-source "${SRC}"/lib/functions/general/file-mtime.sh
-
 # no errors tolerated. invoked before each sourced file to make sure.
 #set -o pipefail  # trace ERR through pipes - will be enabled "soon"
 #set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled