don't ship qemu binary in rootfs cache + armhf image/rootfs building on non-armhf-capable-arm64 hosts (Apple M1+)

- refactor `prepare_host_binfmt_qemu()` out of `prepare_host_noninteractive()` and into `rootfs/qemu-static.sh`
  - further split into more functions and return early to avoid deep nesting
  - implement force import and load of qemu-arm for non-armhf capable arm64 hosts (incl magic numbers)
- enhance `deploy_qemu_binary_to_chroot()` & `undeploy_qemu_binary_from_chroot()`;
  - add 2nd param "caller" for better logging/tracking
  - does sanity-check and preserve existing binary if it exists
- explicitly deploy/undeploy for the 3 cases:
  - image: moved undeploy from `post_debootstrap_tweaks()` into image build proper for consistency
  - rootfs: was leaving trash behind (since post_debootstrap_tweaks never ran for rootfs), now properly undeploys
  - initrd: was already fine, just added caller info
- added `arch-test` host dependency
  - ensure `arch-test ${ARCH}` works during prepare-host
- > tl,dr: "can build 32-bit armv7 armhf using Apple silicon; can use rootfs cache cross-arch reliably"

lib/functions/host/prepare-host.sh

@@ -117,80 +117,7 @@ function prepare_host_noninteractive() {
 		download_external_toolchains # Mostly deprecated, since SKIP_EXTERNAL_TOOLCHAINS=yes is the default
 	fi
 
-	# NEEDS_BINFMT=yes is set by default build and rootfs artifact build.
+	prepare_host_binfmt_qemu # in qemu-static.sh as most binfmt/qemu logic is there now
-	# if we're building an image, not only packages/artifacts...
-	# ... and the host arch does not match the target arch ...
-	# ... we then require binfmt_misc to be enabled.
-	# "enable arm binary format so that the cross-architecture chroot environment will work"
-	if [[ "${NEEDS_BINFMT:-"no"}" == "yes" ]]; then
-
-		if [[ "${SHOW_DEBUG}" == "yes" ]]; then
-			display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug"
-			run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
-		fi
-
-		if dpkg-architecture -e "${ARCH}"; then
-			display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit"
-		else
-			local failed_binfmt_modprobe=0
-
-			display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "debug"
-
-			# Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in.
-			if grep -q "^binfmt_misc" /proc/modules; then
-				display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug"
-			else
-				display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug"
-
-				# try to modprobe. if it fails, emit a warning later, but not here.
-				# this is for the in-container case, where the host already has the module, but won't let the container know about it.
-				modprobe -q binfmt_misc || failed_binfmt_modprobe=1
-			fi
-
-			# Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message
-			if mountpoint -q /proc/sys/fs/binfmt_misc/; then
-				display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug"
-			else
-				display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug"
-				mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || {
-					if [[ $failed_binfmt_modprobe == 1 ]]; then
-						display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn"
-					fi
-					display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn"
-					display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err"
-					exit_with_error "Failed to mount binfmt_misc"
-				}
-				display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug"
-			fi
-
-			declare host_arch
-			host_arch="$(arch)"
-			local -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64")
-			display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug"
-			declare wanted_arch
-			for wanted_arch in "${wanted_arches[@]}"; do
-				if [[ "${host_arch}" != "${wanted_arch}" ]]; then
-					if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" ]]; then
-						display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug"
-						if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then
-							display_alert "Trying to update binfmts - aarch64 (sometimes) does 32-bit sans emulation" "update-binfmts --enable qemu-${wanted_arch}" "debug"
-							run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading.
-						else
-							run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure
-						fi
-					fi
-				fi
-			done
-
-			# @TODO: we could create a tiny loop here to test if the binfmt_misc is working, but this is before deps are installed.
-		fi
-
-		if [[ "${SHOW_DEBUG}" == "yes" ]]; then
-			display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug"
-			run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
-		fi
-
-	fi
 
 	# @TODO: rpardini: this does not belong here, instead with the other templates, pre-configuration.
 	[[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && run_host_command_logged cp -pv "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh
@@ -267,7 +194,7 @@ function adaptative_prepare_host_dependencies() {
 		ncurses-base ncurses-term # for `make menuconfig`
 		ntpdate
 		patchutils pkg-config pv
-		qemu-user-static
+		"qemu-user-static" "arch-test"
 		rsync
 		swig # swig is needed for some u-boot's. example: "bananapi.conf"
 		u-boot-tools

lib/functions/image/initrd.sh

@@ -75,7 +75,7 @@ update_initramfs() {
 	display_alert "initrd cache hash" "${initrd_hash}" "debug"
 
 	display_alert "Mounting chroot for update-initramfs" "update-initramfs" "debug"
-	deploy_qemu_binary_to_chroot "${chroot_target}"
+	deploy_qemu_binary_to_chroot "${chroot_target}" "initrd"# is undeployed at the end of this function
 
 	mount_chroot "$chroot_target/"
 
@@ -125,7 +125,7 @@ update_initramfs() {
 
 	display_alert "Unmounting chroot" "update-initramfs" "debug"
 	umount_chroot "${chroot_target}/"
-	undeploy_qemu_binary_from_chroot "${chroot_target}"
+	undeploy_qemu_binary_from_chroot "${chroot_target}" "initrd" # deployed at the start of this function
 
 	# no need to remove ${initrd_cache_current_manifest_filepath} manually, since it's under ${WORKDIR}
 	return 0 # avoid future short-circuit problems

lib/functions/main/rootfs-image.sh

@@ -13,6 +13,9 @@ function build_rootfs_and_image() {
 	# get a basic rootfs, either from cache or from scratch
 	get_or_create_rootfs_cache_chroot_sdcard # only occurrence of this; has its own logging sections
 
+	# deploy the qemu binary, no matter where the rootfs came from (built or cached)
+	LOG_SECTION="deploy_qemu_binary_to_chroot_image" do_with_logging deploy_qemu_binary_to_chroot "${SDCARD}" "image" # undeployed at end of this function
+
 	# stage: with a basic rootfs available, we mount the chroot and work on it
 	LOG_SECTION="mount_chroot_sdcard" do_with_logging mount_chroot "${SDCARD}"
 
@@ -66,6 +69,9 @@ function build_rootfs_and_image() {
 
 	LOG_SECTION="post_debootstrap_tweaks" do_with_logging post_debootstrap_tweaks
 
+	# undeploy the qemu binary from the image; we don't want to ship the host's qemu in the target image
+	LOG_SECTION="undeploy_qemu_binary_from_chroot_image" do_with_logging undeploy_qemu_binary_from_chroot "${SDCARD}" "image"
+
 	# clean up / prepare for making the image
 	LOG_SECTION="umount_chroot_sdcard" do_with_logging umount_chroot "${SDCARD}"
 

lib/functions/rootfs/post-tweaks.sh

@@ -25,12 +25,10 @@ function post_debootstrap_tweaks() {
 	chroot_sdcard dpkg-divert --quiet --local --rename --remove /sbin/start-stop-daemon
 	run_host_command_logged rm -fv "${SDCARD}"/usr/sbin/policy-rc.d
 
-	# remove the qemu static binary
-	undeploy_qemu_binary_from_chroot "${SDCARD}"
-
 	call_extension_method "post_post_debootstrap_tweaks" "config_post_debootstrap_tweaks" <<- 'POST_POST_DEBOOTSTRAP_TWEAKS'
 		*run after removing diversions and qemu with chroot unmounted*
 		Last chance to touch the `${SDCARD}` filesystem before it is copied to the final media.
-		It is too late to run any chrooted commands, since the supporting filesystems are already unmounted.
 	POST_POST_DEBOOTSTRAP_TWEAKS
+
+	return 0
 }

lib/functions/rootfs/qemu-static.sh

@@ -8,30 +8,185 @@
 # https://github.com/armbian/build/
 
 function deploy_qemu_binary_to_chroot() {
-	local chroot_target="${1}"
+	declare chroot_target="${1}" caller="${2}"
+	display_alert "deploy_qemu_binary_to_chroot" "deploy_qemu_binary_to_chroot '${chroot_target}' during '${caller}'" "debug"
 
-	# @TODO: rpardini: Only deploy the binary if we're actually building a different architecture? otherwise unneeded.
+	# Only deploy the binary if we're actually building a non-native architecture.
-
+	if dpkg-architecture -e "${ARCH}"; then
-	if [[ ! -f "${chroot_target}/usr/bin/${QEMU_BINARY}" ]]; then
+		display_alert "Native build" "not deploying qemu binary during ${caller}" "debug"
-		display_alert "Deploying qemu-user-static binary to chroot" "${QEMU_BINARY}" "debug"
-		run_host_command_logged cp -pv "/usr/bin/${QEMU_BINARY}" "${chroot_target}/usr/bin/"
-	else
-		display_alert "qemu-user-static binary already deployed, skipping" "${QEMU_BINARY}" "debug"
-	fi
-}
-
-function undeploy_qemu_binary_from_chroot() {
-	local chroot_target="${1}"
-
-	# Hack: Check for magic "/usr/bin/qemu-s390x-static" marker; if that exists, it means "qemu-user-static" was installed
-	# in the chroot, and we shouldn't remove the binary, otherwise it's gonna be missing in the final image.
-	if [[ -f "${chroot_target}/usr/bin/qemu-s390x-static" ]]; then
-		display_alert "Not removing qemu binary, qemu-user-static package is installed in the chroot" "${QEMU_BINARY}" "debug"
 		return 0
 	fi
 
-	if [[ -f "${chroot_target}/usr/bin/${QEMU_BINARY}" ]]; then
+	declare src_host="/usr/bin/${QEMU_BINARY}"
-		display_alert "Removing qemu-user-static binary from chroot" "${QEMU_BINARY}" "debug"
+	declare dst_target="${chroot_target}/usr/bin/${QEMU_BINARY}"
-		run_host_command_logged rm -fv "${chroot_target}/usr/bin/${QEMU_BINARY}"
+	declare dst_target_bkp="${dst_target}.armbian.orig"
+
+	# Assume we're getting a clean base to work with. Namely, we count on the rootfs cache to _not_ have left a dangling binary.
+	# If the dst_target already exists, it means the target actually has the qemu-static package installed.
+	# In that case, we preserve the original binary; it will be restored by the undeploy.
+	if [[ -f "${dst_target}" ]]; then
+		display_alert "Preserving existing qemu binary" "${QEMU_BINARY} during ${caller}" "info"
+		run_host_command_logged mv -v "${dst_target}" "${dst_target_bkp}"
+	fi
+
+	display_alert "Deploying qemu-user-static binary to chroot" "${QEMU_BINARY} during ${caller}" "info"
+	run_host_command_logged cp -pv "${src_host}" "${dst_target}"
+
+	return 0
+}
+
+function undeploy_qemu_binary_from_chroot() {
+	declare chroot_target="${1}" caller="${2}"
+	display_alert "undeploy_qemu_binary_from_chroot" "undeploy_qemu_binary_from_chroot '${chroot_target}' during '${caller}'" "debug"
+
+	# Only deploy the binary if we're actually building a non-native architecture.
+	if dpkg-architecture -e "${ARCH}"; then
+		display_alert "Native build" "not deploying qemu binary during ${caller}" "debug"
+		return 0
+	fi
+
+	declare dst_target="${chroot_target}/usr/bin/${QEMU_BINARY}"
+	declare dst_target_bkp="${dst_target}.armbian.orig"
+
+	# Check the binary we deployed is there. If not, panic, as we've lost control.
+	if [[ ! -f "${dst_target}" ]]; then
+		exit_with_error "Missing qemu binary during undeploy_qemu_binary_from_chroot from ${caller}"
+	fi
+
+	# Remove the binary we deployed, and restore the original if we had to preserve it.
+	display_alert "Removing qemu-user-static binary from chroot" "${QEMU_BINARY} during ${caller}" "info"
+	run_host_command_logged rm -fv "${dst_target}"
+
+	if [[ -f "${dst_target_bkp}" ]]; then
+		display_alert "Restoring original qemu binary" "${QEMU_BINARY} during ${caller}" "info"
+		run_host_command_logged mv -v "${dst_target_bkp}" "${dst_target}"
+	fi
+
+	return 0
+}
+
+# "enable arm binary format so that the cross-architecture chroot environment will work" - classic comment from 2013
+# this is called from prepare-host.sh::prepare_host_noninteractive() unconditionally.
+function prepare_host_binfmt_qemu() {
+	# NEEDS_BINFMT=yes is set by "default build" (image build) and rootfs artifact build, which is what requires binfmt_misc to be working.
+	if [[ "${NEEDS_BINFMT:-"no"}" != "yes" ]]; then
+		return 0
+	fi
+
+	if [[ "${SHOW_DEBUG}" == "yes" ]]; then
+		display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug"
+		run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
+	fi
+
+	if dpkg-architecture -e "${ARCH}"; then
+		display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit"
+	else
+		prepare_host_binfmt_qemu_cross
+	fi
+
+	if [[ "${SHOW_DEBUG}" == "yes" ]]; then
+		display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug"
+		run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
+	fi
+
+	# Actually test, using `arch-test`, that we can run binaries for the wanted architecture.
+	display_alert "checking" "arch-test for '${ARCH}'" "info"
+	run_host_command_logged arch-test "${ARCH}"
+
+	# Everything should be either setup or previously correct if we get here.
+	return 0
+}
+
+# The actual binfmt manipulations when cross-build is confirmed above.
+function prepare_host_binfmt_qemu_cross() {
+	local failed_binfmt_modprobe=0
+
+	display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "info"
+
+	# Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in.
+	if grep -q "^binfmt_misc" /proc/modules; then
+		display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug"
+	else
+		display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug"
+
+		# try to modprobe. if it fails, emit a warning later, but not here.
+		# this is for the in-container case, where the host already has the module, but won't let the container know about it.
+		modprobe -q binfmt_misc || failed_binfmt_modprobe=1
+	fi
+
+	# Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message
+	if mountpoint -q /proc/sys/fs/binfmt_misc/; then
+		display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug"
+	else
+		display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug"
+		mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || {
+			if [[ $failed_binfmt_modprobe == 1 ]]; then
+				display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn"
+			fi
+			display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn"
+			display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err"
+			exit_with_error "Failed to mount binfmt_misc"
+		}
+		display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug"
+	fi
+
+	declare host_arch
+	host_arch="$(arch)"
+	declare -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64")
+	declare -A arch_aliases=(["aarch64"]="arm64" ["x86_64"]="amd64")
+	display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug"
+	declare wanted_arch arch_alias
+	for wanted_arch in "${wanted_arches[@]}"; do
+		arch_alias="${arch_aliases["${wanted_arch}"]:-"${wanted_arch}"}"
+		display_alert "Preparing binfmts for arch" "wanted arch '${wanted_arch}' alias '${arch_alias}'" "debug"
+
+		if [[ "${host_arch}" == "${wanted_arch}" ]]; then
+			continue
+		fi
+
+		if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" || ! -e "/usr/share/binfmts/qemu-${wanted_arch}" ]]; then
+			display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug"
+
+			# special case: some arm64 machines cant' really run armhf binaries natively (Apple Silicon); check if that is the case and forcibly import and enable qemu-arm for them.
+			if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then
+				prepare_host_binfmt_qemu_cross_arm64_host_armhf_target
+			else
+				run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure
+			fi
+		fi
+	done
+}
+
+function prepare_host_binfmt_qemu_cross_arm64_host_armhf_target() {
+	display_alert "Trying to update binfmts - aarch64 mostly does 32-bit sans emulation, but Apple said no" "update-binfmts --enable qemu-${wanted_arch}" "debug"
+	run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading.
+
+	if [[ "${SHOW_DEBUG}" == "yes" ]]; then
+		display_alert "Debugging arch-test" "full output" "debug"
+		run_host_command_logged arch-test "||" true
+	fi
+
+	# to check, we use arch-test; if will return 0 if _either_ the host can natively run armhf, or if qemu-arm is correctly working.
+	if arch-test arm; then
+		display_alert "Host can run armhf natively or emulation is correctly setup already" "no need to enable qemu-arm" "debug"
+	else
+		display_alert "arm64 host can't run armhf natively" "importing enabling qemu-arm" "debug"
+		cat <<-BINFMT_ARM_MAGIC >/usr/share/binfmts/qemu-arm
+			package qemu-user-static
+			interpreter /usr/bin/qemu-arm-static
+			magic \x7f\x45\x4c\x46\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00
+			offset 0
+			mask \xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff
+			credentials yes
+			fix_binary no
+			preserve yes
+		BINFMT_ARM_MAGIC
+		run_host_command_logged update-binfmts --import "qemu-${wanted_arch}"
+		run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}"
+
+		# Test again using arch-test.
+		display_alert "Checking if arm 32-bit emulation on arm64 works after enabling" "qemu-arm emulation" "info"
+		run_host_command_logged arch-test arm
+		display_alert "arm 32-bit emulation on arm64" "has been correctly setup" "cachehit"
 	fi
 }

lib/functions/rootfs/rootfs-create.sh

@@ -121,7 +121,7 @@ function create_new_rootfs_cache_via_debootstrap() {
 
 	skip_target_check="yes" local_apt_deb_cache_prepare "after debootstrap first stage" # just for size reference in logs; skip the target check: debootstrap uses it for second stage.
 
-	deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache
+	deploy_qemu_binary_to_chroot "${SDCARD}" "rootfs" # undeployed near the end of this function
 
 	display_alert "Installing base system" "Stage 2/2" "info"
 	declare -g -a if_error_find_files_sdcard=("debootstrap.log") # if command fails, go look for this file and show it's contents during error processing
@@ -270,6 +270,9 @@ function create_new_rootfs_cache_via_debootstrap() {
 	# `armbian-first-run` will do the same thing later
 	chroot_sdcard systemctl mask systemd-firstboot.service
 
+	# undeploy the qemu binary; we don't want to ship the host's qemu binary in the rootfs cache.
+	undeploy_qemu_binary_from_chroot "${SDCARD}" "rootfs"
+
 	# stage: make rootfs cache archive
 	display_alert "Ending debootstrap process and preparing cache" "$RELEASE" "info"
 	wait_for_disk_sync "before tar rootfs"