rootfs/image: introduce new hook custom_apt_repo() (hashed into rootfs version); deploy different repo components/custom repos depending on rootfs or image

- rationale: we don't want an eternal chicken-egg problem with rootfs vs repo.
  - but, desktop rootfs require some parts of repo. case in point: `system-monitoring-center`
    - so only add certain components of repo (-desktop, -utils) to rootfs so that is honored
- introduce `custom_apt_repo()` hook for extensions to add their repos as well
  - same chicken-egg-avoiding is possible via param `CUSTOM_REPO_WHEN`

lib/functions/artifacts/artifact-rootfs.sh

@@ -32,7 +32,7 @@ function artifact_rootfs_prepare_version() {
 
 	calculate_rootfs_cache_id # sets rootfs_cache_id
 
-	display_alert "Going to build rootfs" "packages_hash: '${packages_hash:-}' cache_type: '${cache_type:-}' rootfs_cache_id: '${rootfs_cache_id}'" "info"
+	display_alert "rootfs version" "packages_hash: '${packages_hash:-}' cache_type: '${cache_type:-}' rootfs_cache_id: '${rootfs_cache_id}'" "info"
 
 	declare -a reasons=(
 		"arch \"${ARCH}\""

lib/functions/rootfs/create-cache.sh

@@ -16,15 +16,23 @@ function calculate_rootfs_cache_id() {
 	[[ "x${packages_hash}x" != "xx" ]] && exit_with_error "packages_hash is already set"
 	[[ "x${cache_type}x" != "xx" ]] && exit_with_error "cache_type is already set"
 
+	declare -i short_hash_size=6
+
 	# get the hashes of the lib/ bash sources involved...
 	declare hash_files="undetermined"
 	calculate_hash_for_files "${SRC}"/lib/functions/rootfs/create-cache.sh "${SRC}"/lib/functions/rootfs/rootfs-create.sh
 	declare bash_hash="${hash_files}"
-	declare bash_hash_short="${bash_hash:0:6}"
+	declare bash_hash_short="${bash_hash:0:${short_hash_size}}"
+
+	# hash hooks that affect the rootfs
+	declare -a extension_hooks_to_hash=("custom_apt_repo")
+	declare -a extension_hooks_hashed=("$(dump_extension_method_sources_functions "${extension_hooks_to_hash[@]}")")
+	declare hash_hooks="undetermined"
+	hash_hooks="$(echo "${extension_hooks_hashed[@]}" | sha256sum | cut -d' ' -f1)"
+	declare hash_hooks_short="${hash_hooks:0:${short_hash_size}}"
 
 	# AGGREGATED_ROOTFS_HASH is produced by aggregation.py
-	# Don't use a dash here, dashes are significant to legacy rootfs cache id's
-	declare -g -r packages_hash="${AGGREGATED_ROOTFS_HASH:0:12}B${bash_hash_short}"
+	declare -g -r packages_hash="${AGGREGATED_ROOTFS_HASH:0:12}-H${hash_hooks_short}-B${bash_hash_short}"
 
 	declare cache_type="cli"
 	[[ ${BUILD_DESKTOP} == yes ]] && cache_type="xfce-desktop"
@@ -116,7 +124,9 @@ function extract_rootfs_artifact() {
 	run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf
 	run_host_command_logged echo "nameserver ${NAMESERVER}" ">" "${SDCARD}"/etc/resolv.conf
 
-	create_sources_list "${RELEASE}" "${SDCARD}/"
+	# all sources etc.
+	# armbian repo is fully included, inclusive the components that have debs produced by armbian/build.
+	create_sources_list_and_deploy_repo_key "image" "${RELEASE}" "${SDCARD}/"
 
 	return 0
 }

lib/functions/rootfs/distro-specific.sh

@@ -85,15 +85,17 @@ function install_distribution_specific() {
 	fi
 }
 
-# create_sources_list <release> <basedir>
+# create_sources_list_and_deploy_repo_key <when> <release> <basedir>
 #
+# <when>: rootfs|image
 # <release>: bullseye|bookworm|sid|focal|jammy|kinetic|lunar
 # <basedir>: path to root directory
 #
-function create_sources_list() {
-	local release=$1
-	local basedir=$2 # @TODO: rpardini: this is SDCARD in all practical senses. Why not just use SDCARD?
-	[[ -z $basedir ]] && exit_with_error "No basedir passed to create_sources_list"
+function create_sources_list_and_deploy_repo_key() {
+	declare when="${1}"
+	declare release="${2}"
+	declare basedir="${3}" # @TODO: rpardini: this is SDCARD in all practical senses. Why not just use SDCARD?
+	[[ -z $basedir ]] && exit_with_error "No basedir passed to create_sources_list_and_deploy_repo_key"
 
 	case $release in
 		buster)
@@ -172,7 +174,7 @@ function create_sources_list() {
 			;;
 	esac
 
-	display_alert "Adding Armbian repository and authentication key" "/etc/apt/sources.list.d/armbian.list" "info"
+	display_alert "Adding Armbian repository and authentication key" "${when} :: /etc/apt/sources.list.d/armbian.list" "info"
 
 	# apt-key add is getting deprecated
 	APT_VERSION=$(chroot "${basedir}" /bin/bash -c "apt --version | cut -d\" \" -f2")
@@ -188,17 +190,24 @@ function create_sources_list() {
 		chroot "${basedir}" /bin/bash -c "cat armbian.key | apt-key add -"
 	fi
 
+	declare -a components=()
+	if [[ "${when}" == "image" ]]; then # only include the 'main' component when deploying to image
+		components+=("main")
+	fi
+	components+=("${RELEASE}-utils")   # utils contains packages Igor picks from other repos
+	components+=("${RELEASE}-desktop") # desktop contains packages Igor picks from other repos
+
 	# stage: add armbian repository and install key
 	if [[ $DOWNLOAD_MIRROR == "china" ]]; then
-		echo "deb ${SIGNED_BY}https://mirrors.tuna.tsinghua.edu.cn/armbian $RELEASE main ${RELEASE}-utils ${RELEASE}-desktop" > "${basedir}"/etc/apt/sources.list.d/armbian.list
+		echo "deb ${SIGNED_BY}https://mirrors.tuna.tsinghua.edu.cn/armbian $RELEASE ${components[*]}" > "${basedir}"/etc/apt/sources.list.d/armbian.list
 	elif [[ $DOWNLOAD_MIRROR == "bfsu" ]]; then
-		echo "deb ${SIGNED_BY}http://mirrors.bfsu.edu.cn/armbian $RELEASE main ${RELEASE}-utils ${RELEASE}-desktop" > "${basedir}"/etc/apt/sources.list.d/armbian.list
+		echo "deb ${SIGNED_BY}http://mirrors.bfsu.edu.cn/armbian $RELEASE ${components[*]}" > "${basedir}"/etc/apt/sources.list.d/armbian.list
 	else
-		echo "deb ${SIGNED_BY}http://$([[ $BETA == yes ]] && echo "beta" || echo "apt").armbian.com $RELEASE main ${RELEASE}-utils ${RELEASE}-desktop" > "${basedir}"/etc/apt/sources.list.d/armbian.list
+		echo "deb ${SIGNED_BY}http://$([[ $BETA == yes ]] && echo "beta" || echo "apt").armbian.com $RELEASE ${components[*]}" > "${basedir}"/etc/apt/sources.list.d/armbian.list
 	fi
 
 	# replace local package server if defined. Suitable for development
-	[[ -n $LOCAL_MIRROR ]] && echo "deb ${SIGNED_BY}http://$LOCAL_MIRROR $RELEASE main ${RELEASE}-utils ${RELEASE}-desktop" > "${basedir}"/etc/apt/sources.list.d/armbian.list
+	[[ -n $LOCAL_MIRROR ]] && echo "deb ${SIGNED_BY}http://$LOCAL_MIRROR $RELEASE ${components[*]}" > "${basedir}"/etc/apt/sources.list.d/armbian.list
 
 	# disable repo if SKIP_ARMBIAN_REPO=yes
 	if [[ "${SKIP_ARMBIAN_REPO}" == "yes" ]]; then
@@ -206,4 +215,19 @@ function create_sources_list() {
 		mv "${SDCARD}"/etc/apt/sources.list.d/armbian.list "${SDCARD}"/etc/apt/sources.list.d/armbian.list.disabled
 	fi
 
+	declare CUSTOM_REPO_WHEN="${when}"
+
+	# Let user customize
+	call_extension_method "custom_apt_repo" <<- 'CUSTOM_APT_REPO'
+		*customize apt sources.list.d and/or deploy repo keys*
+		Called after core Armbian has finished setting up SDCARD's sources.list and sources.list.d/armbian.list.
+		If SKIP_ARMBIAN_REPO=yes, armbian.list.disabled is present instead.
+		The global Armbian GPG key has been deployed to SDCARD's /usr/share/keyrings/armbian.gpg, de-armored.
+		You can implement this hook to add, remove, or modify sources.list.d entries, and/or deploy additional GPG keys.
+		Important: honor $CUSTOM_REPO_WHEN; if it's ==rootfs, don't add repos/components that carry the .debs produced by armbian/build.
+	CUSTOM_APT_REPO
+
+	unset CUSTOM_REPO_WHEN
+
+	return 0
 }

lib/functions/rootfs/rootfs-create.sh

@@ -145,8 +145,9 @@ function create_new_rootfs_cache_via_debootstrap() {
 		chroot_sdcard LC_ALL=C LANG=C setupcon --save --force
 	fi
 
-	# stage: create apt-get sources list (basic Debian/Ubuntu apt sources, no external nor PPAS)
-	create_sources_list "$RELEASE" "$SDCARD/"
+	# stage: create apt-get sources list (basic Debian/Ubuntu apt sources, no external nor PPAS).
+	# for the Armbian repo, only the components which are _not_ produced by armbian/build are included (-desktop and -utils)
+	create_sources_list_and_deploy_repo_key "root" "$RELEASE" "$SDCARD/"
 
 	# optionally add armhf arhitecture to arm64, if asked to do so.
 	if [[ "a${ARMHF_ARCH}" == "ayes" ]]; then