rootfs/image: avoid apt junk leftover by debootstrap; add warnings and debugs over target cache and general rootfs sizes

- rootfs: rootfs-create: show a summary of the 20 biggest dirs, right before tarring the rootfs (for debugging)
- rootfs: rootfs-create: show usage of caches between first and second stages
- rootfs: rootfs-create: cleanup junk left by `debootstrap` after second stage
- rootfs: rootfs-create: _always_ clean apt stuff at the end
- rename `apt_purge_unneeded_packages()` to `apt_purge_unneeded_packages_and_clean_apt_caches()` for clarity
- image: `apt_purge_unneeded_packages_and_clean_apt_caches()`: warn if apt caches not empty; clean them off, always.
- host-utils: `local_apt_deb_cache_prepare()`: also test the target, warn if not empty
- extension: cleanup-space-final-image: do NOT clean apt stuff. done in core now
- the metric shit-ton of debugs added should help the next person who faces this in the future

extensions/cleanup-space-final-image.sh

@@ -3,8 +3,7 @@ function add_host_dependencies__cleanup_space_final_image_zerofree() {
 }
 
 function post_customize_image__998_cleanup_apt_stuff() {
-	display_alert "Cleaning up apt package lists and cache" "${EXTENSION}" "info"
-	chroot_sdcard "apt-get clean && rm -rf /var/lib/apt/lists"
+	# This used to clean apt caches, but no longer; we do that in the core now.
 
 	declare -a too_big_firmware=("netronome" "qcom" "mrv" "qed" "mellanox") # maybe: "amdgpu" "radeon" but I have an AMD GPU.
 	for big_firm in "${too_big_firmware[@]}"; do

lib/functions/host/host-utils.sh

@@ -165,6 +165,32 @@ function local_apt_deb_cache_prepare() {
 		[LAST_USED]="${when_used}"
 	)
 
+	if [[ "${skip_target_check:-"no"}" != "yes" ]]; then
+		# Lets take the chance here and _warn_ if the _target_ cache is not empty. Skip if dir doesn't exist or is a mountpoint.
+		declare sdcard_var_cache_apt_dir="${SDCARD}/var/cache/apt"
+		if [[ -d "${sdcard_var_cache_apt_dir}" ]]; then
+			if ! mountpoint -q "${sdcard_var_cache_apt_dir}"; then
+				declare -i sdcard_var_cache_apt_size_mb
+				sdcard_var_cache_apt_size_mb=$(du -sm "${sdcard_var_cache_apt_dir}" | cut -f1)
+				if [[ "${sdcard_var_cache_apt_size_mb}" -gt 0 ]]; then
+					display_alert "WARNING: SDCARD /var/cache/apt dir is not empty" "${when_used} :: ${sdcard_var_cache_apt_dir} (${sdcard_var_cache_apt_size_mb} MB)" "wrn"
+				fi
+			fi
+		fi
+
+		# Same, but for /var/lib/apt/lists
+		declare sdcard_var_lib_apt_lists_dir="${SDCARD}/var/lib/apt/lists"
+		if [[ -d "${sdcard_var_lib_apt_lists_dir}" ]]; then
+			if ! mountpoint -q "${sdcard_var_lib_apt_lists_dir}"; then
+				declare -i sdcard_var_lib_apt_lists_size_mb
+				sdcard_var_lib_apt_lists_size_mb=$(du -sm "${sdcard_var_lib_apt_lists_dir}" | cut -f1)
+				if [[ "${sdcard_var_lib_apt_lists_size_mb}" -gt 0 ]]; then
+					display_alert "WARNING: SDCARD /var/lib/apt/lists dir is not empty" "${when_used} :: ${sdcard_var_lib_apt_lists_dir} (${sdcard_var_lib_apt_lists_size_mb} MB)" "wrn"
+				fi
+			fi
+		fi
+	fi
+
 	return 0
 }
 

lib/functions/main/rootfs-image.sh

@@ -40,7 +40,7 @@ function build_rootfs_and_image() {
 	LOG_SECTION="customize_image" do_with_logging customize_image
 
 	# remove packages that are no longer needed. rootfs cache + uninstall might have leftovers.
-	LOG_SECTION="apt_purge_unneeded_packages" do_with_logging apt_purge_unneeded_packages
+	LOG_SECTION="apt_purge_unneeded_packages_and_clean_apt_caches" do_with_logging apt_purge_unneeded_packages_and_clean_apt_caches
 
 	# for reference, debugging / sanity checking
 	LOG_SECTION="list_installed_packages" do_with_logging list_installed_packages

lib/functions/rootfs/apt-install.sh

@@ -7,10 +7,47 @@
 # This file is a part of the Armbian Build Framework
 # https://github.com/armbian/build/
 
-function apt_purge_unneeded_packages() {
+function apt_purge_unneeded_packages_and_clean_apt_caches() {
 	# remove packages that are no longer needed. rootfs cache + uninstall might have leftovers.
 	display_alert "No longer needed packages" "purge" "info"
 	chroot_sdcard_apt_get autoremove
+
+	declare dir_var_lib_apt_lists="/var/lib/apt/lists"
+	declare dir_var_cache_apt="/var/cache/apt"
+	declare -i dir_var_cache_apt_size_mb dir_var_cache_apt_size_after_cleaning_mb dir_var_lib_apt_lists_size_mb
+
+	# Now, let's list what is under ${SDCARD}/var/cache/apt -- it should be empty. If it isn't, warn, and clean it up.
+	dir_var_cache_apt_size_mb="$(du -sm "${SDCARD}${dir_var_cache_apt}" | cut -f1)"
+	if [[ "${dir_var_cache_apt_size_mb}" -gt 0 ]]; then
+		display_alert "SDCARD ${dir_var_cache_apt} is not empty" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_mb}MB" "wrn"
+		# list the contents
+		run_host_command_logged ls -lahtR "${SDCARD}${dir_var_cache_apt}"
+		wait_for_disk_sync "after listing ${SDCARD}${dir_var_cache_apt}"
+	else
+		display_alert "SDCARD ${dir_var_cache_apt} is empty" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_mb}MB" "debug"
+	fi
+
+	# attention: this is _very different_ from `chroot_sdcard_apt_get clean` (which would clean the cache)
+	chroot_sdcard apt-get clean
+	wait_for_disk_sync "after apt-get clean"
+
+	dir_var_cache_apt_size_after_cleaning_mb="$(du -sm "${SDCARD}${dir_var_cache_apt}" | cut -f1)"
+	display_alert "SDCARD ${dir_var_cache_apt} size after cleaning" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_after_cleaning_mb}MB" "debug"
+
+	# Also clean ${SDCARD}/var/lib/apt/lists; this is where the package lists are stored.
+	dir_var_lib_apt_lists_size_mb="$(du -sm "${SDCARD}${dir_var_lib_apt_lists}" | cut -f1)"
+	if [[ "${dir_var_lib_apt_lists_size_mb}" -gt 0 ]]; then
+		display_alert "SDCARD ${dir_var_lib_apt_lists} is not empty" "${dir_var_lib_apt_lists} :: ${dir_var_lib_apt_lists_size_mb}MB" "wrn"
+		# list the contents
+		run_host_command_logged ls -lahtR "${SDCARD}${dir_var_lib_apt_lists}"
+		wait_for_disk_sync "after listing ${SDCARD}${dir_var_cache_apt}"
+	else
+		display_alert "SDCARD ${dir_var_lib_apt_lists} is empty" "${dir_var_lib_apt_lists} :: ${dir_var_lib_apt_lists_size_mb}MB" "debug"
+	fi
+
+	# Either way, clean it away, we don't wanna ship those lists on images or rootfs.
+	run_host_command_logged rm -rf "${SDCARD}${dir_var_lib_apt_lists}"
+	wait_for_disk_sync "after cleaning ${SDCARD}${dir_var_lib_apt_lists}"
 }
 
 # this is called:

lib/functions/rootfs/rootfs-create.sh

@@ -20,6 +20,11 @@ function create_new_rootfs_cache_tarball() {
 	chroot_sdcard "dpkg -l | grep ^ii | awk '{ print \$2\",\"\$3 }'" > "${cache_fname}.list"
 	echo "${AGGREGATED_ROOTFS_HASH_TEXT}" > "${cache_fname}.hash_text"
 
+	# Show the disk space usage of the rootfs
+	display_alert "Disk space usage of rootfs" "${RELEASE}:: ${cache_name}" "info"
+	run_host_command_logged "cd ${SDCARD} && " du -h -d 4 -x "." "| sort -h | tail -20"
+	wait_for_disk_sync "after disk-space usage report of rootfs"
+
 	declare compression_ratio_rootfs="${ROOTFS_COMPRESSION_RATIO:-"5"}"
 
 	display_alert "zstd tarball of rootfs" "${RELEASE}:: ${cache_name} :: compression ${compression_ratio_rootfs}" "info"
@@ -96,7 +101,7 @@ function create_new_rootfs_cache_via_debootstrap() {
 	}
 	[[ ! -f ${SDCARD}/debootstrap/debootstrap ]] && exit_with_error "Debootstrap first stage did not produce marker file"
 
-	local_apt_deb_cache_prepare "after debootstrap" # just for size reference in logs
+	skip_target_check="yes" local_apt_deb_cache_prepare "after debootstrap first stage" # just for size reference in logs; skip the target check: debootstrap uses it for second stage.
 
 	deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache
 
@@ -105,6 +110,12 @@ function create_new_rootfs_cache_via_debootstrap() {
 	chroot_sdcard LC_ALL=C LANG=C /debootstrap/debootstrap --second-stage
 	[[ ! -f "${SDCARD}/bin/bash" ]] && exit_with_error "Debootstrap first stage did not produce /bin/bash"
 
+	# Done with debootstrap. Clean-up it's litterbox.
+	display_alert "Cleaning up after debootstrap" "debootstrap cleanup" "info"
+	run_host_command_logged rm -rf "${SDCARD}/var/cache/apt" "${SDCARD}/var/lib/apt/lists"
+
+	local_apt_deb_cache_prepare "after debootstrap second stage" # just for size reference in logs
+
 	mount_chroot "${SDCARD}" # we mount the chroot here... it's un-mounted below when all is done, or by cleanup handler '' @TODO
 
 	display_alert "Diverting" "initctl/start-stop-daemon" "info"
@@ -208,11 +219,8 @@ function create_new_rootfs_cache_via_debootstrap() {
 	# don't touch the local cache.
 	DONT_MAINTAIN_APT_CACHE="yes" chroot_sdcard_apt_get autoremove
 
-	# Only clean if not using local cache. Otherwise it would be cleaning the cache, not the chroot.
-	if [[ "${USE_LOCAL_APT_DEB_CACHE}" != "yes" ]]; then
-		display_alert "Late Cleaning" "late: package lists and apt cache" "warn"
-		chroot_sdcard_apt_get clean
-	fi
+	# Purge/clean apt cache in the target. It should _not_ have been used, but if it was, warn & clean.
+	apt_purge_unneeded_packages_and_clean_apt_caches
 
 	# DEBUG: print free space
 	local free_space