diff --git a/extensions/cleanup-space-final-image.sh b/extensions/cleanup-space-final-image.sh index c6e288274..b7d0d8e59 100644 --- a/extensions/cleanup-space-final-image.sh +++ b/extensions/cleanup-space-final-image.sh @@ -3,8 +3,7 @@ function add_host_dependencies__cleanup_space_final_image_zerofree() { } function post_customize_image__998_cleanup_apt_stuff() { - display_alert "Cleaning up apt package lists and cache" "${EXTENSION}" "info" - chroot_sdcard "apt-get clean && rm -rf /var/lib/apt/lists" + # This used to clean apt caches, but no longer; we do that in the core now. declare -a too_big_firmware=("netronome" "qcom" "mrv" "qed" "mellanox") # maybe: "amdgpu" "radeon" but I have an AMD GPU. for big_firm in "${too_big_firmware[@]}"; do diff --git a/lib/functions/host/host-utils.sh b/lib/functions/host/host-utils.sh index 16e9acb83..f9b38ff7b 100644 --- a/lib/functions/host/host-utils.sh +++ b/lib/functions/host/host-utils.sh @@ -165,6 +165,32 @@ function local_apt_deb_cache_prepare() { [LAST_USED]="${when_used}" ) + if [[ "${skip_target_check:-"no"}" != "yes" ]]; then + # Lets take the chance here and _warn_ if the _target_ cache is not empty. Skip if dir doesn't exist or is a mountpoint. + declare sdcard_var_cache_apt_dir="${SDCARD}/var/cache/apt" + if [[ -d "${sdcard_var_cache_apt_dir}" ]]; then + if ! mountpoint -q "${sdcard_var_cache_apt_dir}"; then + declare -i sdcard_var_cache_apt_size_mb + sdcard_var_cache_apt_size_mb=$(du -sm "${sdcard_var_cache_apt_dir}" | cut -f1) + if [[ "${sdcard_var_cache_apt_size_mb}" -gt 0 ]]; then + display_alert "WARNING: SDCARD /var/cache/apt dir is not empty" "${when_used} :: ${sdcard_var_cache_apt_dir} (${sdcard_var_cache_apt_size_mb} MB)" "wrn" + fi + fi + fi + + # Same, but for /var/lib/apt/lists + declare sdcard_var_lib_apt_lists_dir="${SDCARD}/var/lib/apt/lists" + if [[ -d "${sdcard_var_lib_apt_lists_dir}" ]]; then + if ! mountpoint -q "${sdcard_var_lib_apt_lists_dir}"; then + declare -i sdcard_var_lib_apt_lists_size_mb + sdcard_var_lib_apt_lists_size_mb=$(du -sm "${sdcard_var_lib_apt_lists_dir}" | cut -f1) + if [[ "${sdcard_var_lib_apt_lists_size_mb}" -gt 0 ]]; then + display_alert "WARNING: SDCARD /var/lib/apt/lists dir is not empty" "${when_used} :: ${sdcard_var_lib_apt_lists_dir} (${sdcard_var_lib_apt_lists_size_mb} MB)" "wrn" + fi + fi + fi + fi + return 0 } diff --git a/lib/functions/main/rootfs-image.sh b/lib/functions/main/rootfs-image.sh index f40c74e70..fae34acb1 100644 --- a/lib/functions/main/rootfs-image.sh +++ b/lib/functions/main/rootfs-image.sh @@ -40,7 +40,7 @@ function build_rootfs_and_image() { LOG_SECTION="customize_image" do_with_logging customize_image # remove packages that are no longer needed. rootfs cache + uninstall might have leftovers. - LOG_SECTION="apt_purge_unneeded_packages" do_with_logging apt_purge_unneeded_packages + LOG_SECTION="apt_purge_unneeded_packages_and_clean_apt_caches" do_with_logging apt_purge_unneeded_packages_and_clean_apt_caches # for reference, debugging / sanity checking LOG_SECTION="list_installed_packages" do_with_logging list_installed_packages diff --git a/lib/functions/rootfs/apt-install.sh b/lib/functions/rootfs/apt-install.sh index 59ed5e4fe..bbf46546c 100644 --- a/lib/functions/rootfs/apt-install.sh +++ b/lib/functions/rootfs/apt-install.sh @@ -7,10 +7,47 @@ # This file is a part of the Armbian Build Framework # https://github.com/armbian/build/ -function apt_purge_unneeded_packages() { +function apt_purge_unneeded_packages_and_clean_apt_caches() { # remove packages that are no longer needed. rootfs cache + uninstall might have leftovers. display_alert "No longer needed packages" "purge" "info" chroot_sdcard_apt_get autoremove + + declare dir_var_lib_apt_lists="/var/lib/apt/lists" + declare dir_var_cache_apt="/var/cache/apt" + declare -i dir_var_cache_apt_size_mb dir_var_cache_apt_size_after_cleaning_mb dir_var_lib_apt_lists_size_mb + + # Now, let's list what is under ${SDCARD}/var/cache/apt -- it should be empty. If it isn't, warn, and clean it up. + dir_var_cache_apt_size_mb="$(du -sm "${SDCARD}${dir_var_cache_apt}" | cut -f1)" + if [[ "${dir_var_cache_apt_size_mb}" -gt 0 ]]; then + display_alert "SDCARD ${dir_var_cache_apt} is not empty" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_mb}MB" "wrn" + # list the contents + run_host_command_logged ls -lahtR "${SDCARD}${dir_var_cache_apt}" + wait_for_disk_sync "after listing ${SDCARD}${dir_var_cache_apt}" + else + display_alert "SDCARD ${dir_var_cache_apt} is empty" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_mb}MB" "debug" + fi + + # attention: this is _very different_ from `chroot_sdcard_apt_get clean` (which would clean the cache) + chroot_sdcard apt-get clean + wait_for_disk_sync "after apt-get clean" + + dir_var_cache_apt_size_after_cleaning_mb="$(du -sm "${SDCARD}${dir_var_cache_apt}" | cut -f1)" + display_alert "SDCARD ${dir_var_cache_apt} size after cleaning" "${dir_var_cache_apt} :: ${dir_var_cache_apt_size_after_cleaning_mb}MB" "debug" + + # Also clean ${SDCARD}/var/lib/apt/lists; this is where the package lists are stored. + dir_var_lib_apt_lists_size_mb="$(du -sm "${SDCARD}${dir_var_lib_apt_lists}" | cut -f1)" + if [[ "${dir_var_lib_apt_lists_size_mb}" -gt 0 ]]; then + display_alert "SDCARD ${dir_var_lib_apt_lists} is not empty" "${dir_var_lib_apt_lists} :: ${dir_var_lib_apt_lists_size_mb}MB" "wrn" + # list the contents + run_host_command_logged ls -lahtR "${SDCARD}${dir_var_lib_apt_lists}" + wait_for_disk_sync "after listing ${SDCARD}${dir_var_cache_apt}" + else + display_alert "SDCARD ${dir_var_lib_apt_lists} is empty" "${dir_var_lib_apt_lists} :: ${dir_var_lib_apt_lists_size_mb}MB" "debug" + fi + + # Either way, clean it away, we don't wanna ship those lists on images or rootfs. + run_host_command_logged rm -rf "${SDCARD}${dir_var_lib_apt_lists}" + wait_for_disk_sync "after cleaning ${SDCARD}${dir_var_lib_apt_lists}" } # this is called: diff --git a/lib/functions/rootfs/rootfs-create.sh b/lib/functions/rootfs/rootfs-create.sh index 36a214f2c..6a4032c86 100644 --- a/lib/functions/rootfs/rootfs-create.sh +++ b/lib/functions/rootfs/rootfs-create.sh @@ -20,6 +20,11 @@ function create_new_rootfs_cache_tarball() { chroot_sdcard "dpkg -l | grep ^ii | awk '{ print \$2\",\"\$3 }'" > "${cache_fname}.list" echo "${AGGREGATED_ROOTFS_HASH_TEXT}" > "${cache_fname}.hash_text" + # Show the disk space usage of the rootfs + display_alert "Disk space usage of rootfs" "${RELEASE}:: ${cache_name}" "info" + run_host_command_logged "cd ${SDCARD} && " du -h -d 4 -x "." "| sort -h | tail -20" + wait_for_disk_sync "after disk-space usage report of rootfs" + declare compression_ratio_rootfs="${ROOTFS_COMPRESSION_RATIO:-"5"}" display_alert "zstd tarball of rootfs" "${RELEASE}:: ${cache_name} :: compression ${compression_ratio_rootfs}" "info" @@ -96,7 +101,7 @@ function create_new_rootfs_cache_via_debootstrap() { } [[ ! -f ${SDCARD}/debootstrap/debootstrap ]] && exit_with_error "Debootstrap first stage did not produce marker file" - local_apt_deb_cache_prepare "after debootstrap" # just for size reference in logs + skip_target_check="yes" local_apt_deb_cache_prepare "after debootstrap first stage" # just for size reference in logs; skip the target check: debootstrap uses it for second stage. deploy_qemu_binary_to_chroot "${SDCARD}" # this is cleaned-up later by post_debootstrap_tweaks() @TODO: which is too late for a cache @@ -105,6 +110,12 @@ function create_new_rootfs_cache_via_debootstrap() { chroot_sdcard LC_ALL=C LANG=C /debootstrap/debootstrap --second-stage [[ ! -f "${SDCARD}/bin/bash" ]] && exit_with_error "Debootstrap first stage did not produce /bin/bash" + # Done with debootstrap. Clean-up it's litterbox. + display_alert "Cleaning up after debootstrap" "debootstrap cleanup" "info" + run_host_command_logged rm -rf "${SDCARD}/var/cache/apt" "${SDCARD}/var/lib/apt/lists" + + local_apt_deb_cache_prepare "after debootstrap second stage" # just for size reference in logs + mount_chroot "${SDCARD}" # we mount the chroot here... it's un-mounted below when all is done, or by cleanup handler '' @TODO display_alert "Diverting" "initctl/start-stop-daemon" "info" @@ -208,11 +219,8 @@ function create_new_rootfs_cache_via_debootstrap() { # don't touch the local cache. DONT_MAINTAIN_APT_CACHE="yes" chroot_sdcard_apt_get autoremove - # Only clean if not using local cache. Otherwise it would be cleaning the cache, not the chroot. - if [[ "${USE_LOCAL_APT_DEB_CACHE}" != "yes" ]]; then - display_alert "Late Cleaning" "late: package lists and apt cache" "warn" - chroot_sdcard_apt_get clean - fi + # Purge/clean apt cache in the target. It should _not_ have been used, but if it was, warn & clean. + apt_purge_unneeded_packages_and_clean_apt_caches # DEBUG: print free space local free_space