introduce armbian-base-files artifact, which downloads & repacks base-files from upstream distro

- bsp-cli: now depends on `base-files (>= ${REVISION})`, this way upgrading the bsp-cli causes our base-files to be installed
  - bsp-cli no longer does gymnastics with /etc/os-release et al, all done in armbian-base-files now
- general/apt-utils.sh: introduce `apt_find_upstream_package_version_and_download_url()`
- base-files: add release to version, in order to comply with repo restrictions (valid repos can't have two different debs with same name and version, md5 must match)
This commit is contained in:
Ricardo Pardini
2023-05-12 18:11:51 +02:00
committed by igorpecovnik
parent c7fc14a7b8
commit 1fd5d519d4
9 changed files with 318 additions and 32 deletions

View File

@@ -0,0 +1,195 @@
#
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
# This file is a part of the Armbian Build Framework https://github.com/armbian/build/
#
# tl;dr: this artifact is a replacement for the original distro's base-files.
# We find what is the latest version of the original distro's base-files.
# Then we download it, and we modify it to suit our needs.
# The artifact is named "armbian-base-files".
# But the package is still named "base-files"; its version is bumped to ${REVISION}
# This is similar to what Linux Mint does for the same purpose.
function artifact_armbian-base-files_config_dump() {
artifact_input_variables[RELEASE]="${RELEASE}"
artifact_input_variables[ARCH]="${ARCH}"
}
function artifact_armbian-base-files_prepare_version() {
: "${artifact_prefix_version:?artifact_prefix_version is not set}"
: "${RELEASE:?RELEASE is not set}"
: "${ARCH:?ARCH is not set}"
artifact_version="undetermined" # outer scope
artifact_version_reason="undetermined" # outer scope
declare short_hash_size=4
declare fake_unchanging_base_version="${RELEASE}-1armbian1"
declare found_package_version="undetermined" found_package_filename="undetermined" found_package_down_url="undetermined"
sleep_seconds="15" do_with_retries 10 apt_find_upstream_package_version_and_download_url "base-files"
# download the file, but write it to /dev/null (just for testing it is correct)
# wget --timeout=15 --output-document=/dev/null "${found_package_down_url}" || exit_with_error "Could not download '${found_package_down_url}'"
# Set readonly globals with the wanted info; will be used during the actual build of this artifact
declare -g -r base_files_wanted_upstream_version="${found_package_version}"
declare -g -r base_files_wanted_upstream_filename="${found_package_filename}"
declare -g -r base_files_wanted_deb_down_url="${found_package_down_url}"
# get the hashes of the lib/ bash sources involved.
declare hash_files="undetermined"
calculate_hash_for_bash_deb_artifact "artifacts/artifact-armbian-base-files.sh"
declare bash_hash="${hash_files}"
declare bash_hash_short="${bash_hash:0:${short_hash_size}}"
# outer scope
artifact_version="${artifact_prefix_version}${fake_unchanging_base_version}-B${bash_hash_short}"
declare -a reasons=("Armbian armbian-base-files" "original ${RELEASE} version \"${base_files_wanted_upstream_version}\"" "framework bash hash \"${bash_hash}\"")
artifact_version_reason="${reasons[*]}" # outer scope
artifact_name="armbian-base-files-${RELEASE}-${ARCH}"
declare deb_name="base-files" # the artifact_name is only Armbian's reference; the deb_name is still base_files
artifact_type="deb"
artifact_base_dir="${DEB_STORAGE}/${RELEASE}"
artifact_final_file="${DEB_STORAGE}/${RELEASE}/${deb_name}_${artifact_version}_${ARCH}.deb"
artifact_map_packages=(["armbian-base-files"]="${deb_name}")
artifact_map_debs=(["armbian-base-files"]="${RELEASE}/${deb_name}_${artifact_version}_${ARCH}.deb")
return 0
}
function artifact_armbian-base-files_build_from_sources() {
LOG_SECTION="compile_armbian-base-files" do_with_logging compile_armbian-base-files
}
# Dont' wanna use a separate file for this. Keep it in here.
function compile_armbian-base-files() {
: "${artifact_name:?artifact_name is not set}"
: "${artifact_version:?artifact_version is not set}"
: "${RELEASE:?RELEASE is not set}"
: "${ARCH:?ARCH is not set}"
: "${DISTRIBUTION:?DISTRIBUTION is not set}"
display_alert "Creating base-files for ${DISTRIBUTION} release '${RELEASE}' arch '${ARCH}'" "${artifact_name} :: ${artifact_version}" "info"
declare cleanup_id="" destination=""
prepare_temp_dir_in_workdir_and_schedule_cleanup "base-files" cleanup_id destination # namerefs
# Download the deb file
declare deb_file="${destination}/${base_files_wanted_upstream_filename}"
run_host_command_logged wget --no-verbose --timeout=60 --output-document="${deb_file}" "${base_files_wanted_deb_down_url}" || exit_with_error "Could not download '${base_files_wanted_deb_down_url}'"
# Raw-Extract (with DEBIAN dir) the contents of the deb file into "${destination}"
run_host_command_logged dpkg-deb --raw-extract "${deb_file}" "${destination}" || exit_with_error "Could not raw-extract '${deb_file}'"
# Remove the .deb file
rm -f "${deb_file}"
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
# Show the tree
run_host_command_logged tree "${destination}"
wait_for_disk_sync "after tree base-files"
# Show the original control file, using batcat
run_tool_batcat --file-name "${artifact_name}/DEBIAN/control" "${destination}/DEBIAN/control"
# Show the original conffiles file, using batcat
run_tool_batcat --file-name "${artifact_name}/DEBIAN/conffiles" "${destination}/DEBIAN/conffiles"
fi
# Let's hack at it. New Maintainer and Version...
cat <<- EOD >> "${destination}/DEBIAN/control.new"
Maintainer: $MAINTAINER <$MAINTAINERMAIL>
Version: ${artifact_version}
EOD
# Keep everything else from original
cat "${destination}/DEBIAN/control" | grep -vP '^(Maintainer|Version):' >> "${destination}/DEBIAN/control.new"
# Replace 'Debian' with 'Armbian'.
sed -i "s/Debian/${VENDOR}/g" "${destination}/DEBIAN/control.new"
mv "${destination}/DEBIAN/control.new" "${destination}/DEBIAN/control"
# Change etc/os-release, etc/issue, etc/issue.net, and DEBIAN/conffiles
# Keep copies so we can diff
cp "${destination}"/etc/os-release "${destination}"/etc/os-release.orig
cp "${destination}"/etc/issue "${destination}"/etc/issue.orig
cp "${destination}"/etc/issue.net "${destination}"/etc/issue.net.orig
cp "${destination}"/DEBIAN/conffiles "${destination}"/DEBIAN/conffiles.orig
# Change the PRETTY_NAME and add ARMBIAN_PRETTY_NAME in os-release, and change issue, issue.net
declare orig_distro_release="${RELEASE}"
echo "ARMBIAN_PRETTY_NAME=\"${VENDOR} ${REVISION} ${orig_distro_release}\"" >> "${destination}"/etc/os-release
echo -e "${VENDOR} ${REVISION} ${orig_distro_release} \\l \n" > "${destination}"/etc/issue
echo -e "${VENDOR} ${REVISION} ${orig_distro_release}" > "${destination}"/etc/issue.net
sed -i "s/^PRETTY_NAME=.*/PRETTY_NAME=\"${VENDOR} $REVISION ${orig_distro_release}\"/" "${destination}"/etc/os-release
# Remove /etc/issue and /etc/issue.net from the DEBIAN/conffiles file
sed -i '/^\/etc\/issue$/d' "${destination}"/DEBIAN/conffiles
sed -i '/^\/etc\/issue.net$/d' "${destination}"/DEBIAN/conffiles
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
# Show the results of the changing...
run_tool_batcat --file-name "${artifact_name}/etc/os-release" "${destination}"/etc/os-release
run_tool_batcat --file-name "${artifact_name}/etc/issue" "${destination}"/etc/issue
run_tool_batcat --file-name "${artifact_name}/etc/issue.net" "${destination}"/etc/issue.net
run_tool_batcat --file-name "${artifact_name}/DEBIAN/conffiles" "${destination}"/DEBIAN/conffiles
# Show the diffs, use colors.
run_host_command_logged diff --color=always -u "${destination}"/etc/os-release.orig "${destination}"/etc/os-release "||" true
run_host_command_logged diff --color=always -u "${destination}"/etc/issue.orig "${destination}"/etc/issue "||" true
run_host_command_logged diff --color=always -u "${destination}"/etc/issue.net.orig "${destination}"/etc/issue.net "||" true
run_host_command_logged diff --color=always -u "${destination}"/DEBIAN/conffiles.orig "${destination}"/DEBIAN/conffiles "||" true
fi
# Remove the .orig files
rm -f "${destination}"/etc/os-release.orig "${destination}"/etc/issue.orig "${destination}"/etc/issue.net.orig "${destination}"/DEBIAN/conffiles.orig
# Done, pack it.
mkdir -p "${DEB_STORAGE}/${RELEASE}"
fakeroot_dpkg_deb_build "${destination}" "${DEB_STORAGE}/${RELEASE}"
done_with_temp_dir "${cleanup_id}" # changes cwd to "${SRC}" and fires the cleanup function early
}
function artifact_armbian-base-files_cli_adapter_pre_run() {
declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command.
# "gimme root on a Linux machine"
cli_standard_relaunch_docker_or_sudo
}
function artifact_armbian-base-files_cli_adapter_config_prep() {
: "${RELEASE:?RELEASE is not set}"
: "${BOARD:?BOARD is not set}"
# there is no need for aggregation here, although RELEASE is required.
use_board="yes" allow_no_family="no" skip_kernel="no" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
}
function artifact_armbian-base-files_get_default_oci_target() {
artifact_oci_target_base="${GHCR_SOURCE}/armbian/os/"
}
function artifact_armbian-base-files_is_available_in_local_cache() {
is_artifact_available_in_local_cache
}
function artifact_armbian-base-files_is_available_in_remote_cache() {
is_artifact_available_in_remote_cache
}
function artifact_armbian-base-files_obtain_from_remote_cache() {
obtain_artifact_from_remote_cache
}
function artifact_armbian-base-files_deploy_to_remote_cache() {
upload_artifact_to_oci
}

View File

@@ -22,6 +22,7 @@ function armbian_register_artifacts() {
["armbian-config"]="armbian-config"
["armbian-zsh"]="armbian-zsh"
["armbian-plymouth-theme"]="armbian-plymouth-theme"
["armbian-base-files"]="armbian-base-files"
["armbian-bsp-cli"]="armbian-bsp-cli"
["armbian-bsp-desktop"]="armbian-bsp-desktop"
["armbian-desktop"]="armbian-desktop"

View File

@@ -31,6 +31,11 @@ function compile_armbian-bsp-cli() {
# Replaces: base-files is needed to replace /etc/update-motd.d/ files on Xenial
# Depends: linux-base is needed for "linux-version" command in initrd cleanup script
# Depends: fping is needed for armbianmonitor to upload armbian-hardware-monitor.log
# Depends: base-files (>= ${REVISION}) is to force usage of our base-files package (not the original Distro's).
declare depends_base_files=", base-files (>= ${REVISION})"
if [[ "${KEEP_ORIGINAL_OS_RELEASE:-"no"}" == "yes" ]]; then
depends_base_files=""
fi
cat <<- EOF > "${destination}"/DEBIAN/control
Package: ${artifact_name}
Version: ${artifact_version}
@@ -39,7 +44,7 @@ function compile_armbian-bsp-cli() {
Installed-Size: 1
Section: kernel
Priority: optional
Depends: bash, linux-base, u-boot-tools, initramfs-tools, lsb-release, fping
Depends: bash, linux-base, u-boot-tools, initramfs-tools, lsb-release, fping${depends_base_files}
Suggests: armbian-config
Replaces: zram-config, base-files
Recommends: bsdutils, parted, util-linux, toilet
@@ -108,12 +113,7 @@ function compile_armbian-bsp-cli() {
postinst_functions+=(board_side_bsp_cli_postinst_update_uboot_bootscript)
fi
if [[ "${KEEP_ORIGINAL_OS_RELEASE:-"no"}" != "yes" ]]; then
# add to postinst, to change PRETTY_NAME to Armbian's
postinst_functions+=(board_side_bsp_cli_postinst_os_release_armbian)
else
display_alert "bsp-cli: KEEP_ORIGINAL_OS_RELEASE" "Keeping original /etc/os-release's PRETTY_NAME as original" "info"
fi
# PRETTY_NAME stuff is now done in armbian-base-files artifact
# add configuration for setting uboot environment from userspace with: fw_setenv fw_printenv
if [[ -n $UBOOT_FW_ENV ]]; then
@@ -328,14 +328,7 @@ function board_side_bsp_cli_postinst_base() {
# Source the armbian-release information file
[ -f /etc/armbian-release ] && . /etc/armbian-release
# Read release value from lsb-release and set it separately as ARMBIAN_PRETTY_NAME
# More is done, actually taking over PRETTY_NAME, in separate board_side_bsp_cli_postinst_os_release_armbian()
if [ -f /etc/lsb-release ]; then
ORIGINAL_DISTRO_RELEASE="$(cat /etc/lsb-release | grep CODENAME | cut -d"=" -f2 | sed 's/.*/\u&/')"
echo "ARMBIAN_PRETTY_NAME=\"${VENDOR} ${REVISION} ${ORIGINAL_DISTRO_RELEASE}\"" >> /etc/os-release
echo -e "${VENDOR} ${REVISION} ${ORIGINAL_DISTRO_RELEASE} \\l \n" > /etc/issue
echo -e "${VENDOR} ${REVISION} ${ORIGINAL_DISTRO_RELEASE}" > /etc/issue.net
fi
# ARMBIAN_PRETTY_NAME is now set in armbian-base-files.
# Force ramlog to be enabled if it exists. @TODO: why?
[ -f /etc/lib/systemd/system/armbian-ramlog.service ] && systemctl --no-reload enable armbian-ramlog.service
@@ -353,17 +346,6 @@ function board_side_bsp_cli_postinst_base() {
fi
}
function board_side_bsp_cli_postinst_os_release_armbian() {
# Source the armbian-release information file
[ -f /etc/armbian-release ] && . /etc/armbian-release
# Read release value from lsb-release, so deploying a bsp-cli package on top of "X" makes it "Armbian X"
if [ -f /etc/lsb-release ]; then
ORIGINAL_DISTRO_RELEASE="$(cat /etc/lsb-release | grep CODENAME | cut -d"=" -f2 | sed 's/.*/\u&/')"
sed -i "s/^PRETTY_NAME=.*/PRETTY_NAME=\"${VENDOR} $REVISION ${ORIGINAL_DISTRO_RELEASE}\"/" /etc/os-release
fi
}
function board_side_bsp_cli_postinst_finish() {
[ ! -f "/etc/network/interfaces" ] && [ -f "/etc/network/interfaces.default" ] && cp /etc/network/interfaces.default /etc/network/interfaces
ln -sf /var/run/motd /etc/motd

View File

@@ -69,6 +69,7 @@ function armbian_register_commands() {
["armbian-plymouth-theme"]="artifact"
["fake-ubuntu-advantage-tools"]="artifact"
["armbian-base-files"]="artifact"
["armbian-bsp-cli"]="artifact"
["armbian-bsp-desktop"]="artifact"
["armbian-desktop"]="artifact"
@@ -112,6 +113,7 @@ function armbian_register_commands() {
["armbian-plymouth-theme"]="WHAT='armbian-plymouth-theme' ${common_cli_artifact_vars}"
["fake-ubuntu-advantage-tools"]="WHAT='fake_ubuntu_advantage_tools' ${common_cli_artifact_vars}"
["armbian-base-files"]="WHAT='armbian-base-files' ${common_cli_artifact_vars}"
["armbian-bsp-cli"]="WHAT='armbian-bsp-cli' ${common_cli_artifact_vars}"
["armbian-bsp-desktop"]="WHAT='armbian-bsp-desktop' BUILD_DESKTOP='yes' ${common_cli_artifact_vars}"
["armbian-desktop"]="WHAT='armbian-desktop' BUILD_DESKTOP='yes' ${common_cli_artifact_vars}"

View File

@@ -0,0 +1,84 @@
#
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2023 Ricardo Pardini <ricardo@pardini.net>
# This file is a part of the Armbian Build Framework https://github.com/armbian/build/
#
# tl;dr: this function will return the version and download URL of a package.
# it is very naive, and will only look into a few repos across Debian/Ubuntu.
function apt_find_upstream_package_version_and_download_url() {
declare sought_package_name="${1}"
declare -a package_info_download_urls=()
declare first_letter_of_sought_package_name="${sought_package_name:0:1}"
declare mirror_with_slash="undetermined/"
case "${DISTRIBUTION}" in
Ubuntu) # try both the jammy-updates and jammy repos, use whatever returns first
package_info_download_urls+=("https://packages.ubuntu.com/${RELEASE}-updates/${ARCH}/${sought_package_name}/download")
package_info_download_urls+=("https://packages.ubuntu.com/${RELEASE}/${ARCH}/${sought_package_name}/download")
mirror_with_slash="${UBUNTU_MIRROR}"
;;
Debian)
package_info_download_urls+=("https://packages.debian.org/${RELEASE}/${ARCH}/${sought_package_name}/download")
mirror_with_slash="${DEBIAN_MIRROR}"
;;
*)
exit_with_error "Unknown distribution '${DISTRIBUTION}'"
;;
esac
# if mirror_with_slash does not end with a slash, add it
if [[ "${mirror_with_slash}" != */ ]]; then
mirror_with_slash="${mirror_with_slash}/"
fi
declare base_down_url="http://${mirror_with_slash}pool/main/${first_letter_of_sought_package_name}/${sought_package_name}"
declare index package_info_download_url
# loop over the package_info_download_urls with index and value
for index in "${!package_info_download_urls[@]}"; do
package_info_download_url="${package_info_download_urls[$index]}"
display_alert "Testing URL" "${package_info_download_url}" "debug"
declare package_info_download_url_file
package_info_download_url_file="$(mktemp)"
curl --silent --show-error --max-time 10 "${package_info_download_url}" > "${package_info_download_url_file}" || true # don't fail
declare package_info_download_url_file_package_name # grep the file for the package name. parse "<kbd>name</kbd>"
package_info_download_url_file_package_name="$(grep -oP '(?<=<kbd>)[^<]+' "${package_info_download_url_file}" | grep "^${sought_package_name}_" | head -n 1)"
rm -f "${package_info_download_url_file}"
display_alert "Package name parsed" "${package_info_download_url_file_package_name}" "debug"
if [[ "${package_info_download_url_file_package_name}" == "${sought_package_name}_"* ]]; then
found_package_filename="${package_info_download_url_file_package_name}"
found_package_down_url="${base_down_url}/${found_package_filename}"
display_alert "Found package filename" "${found_package_filename} in url ${package_info_download_url}" "debug"
break
fi
done
if [[ "${found_package_filename}" == "${sought_package_name}_"* ]]; then
display_alert "Found upstream base-files package filename" "${found_package_filename}" "info"
else
display_alert "Could not find package filename for '${sought_package_name}' in '${package_info_download_urls[*]}'" "looking for ${sought_package_name}" "warn"
return 1
fi
# Now we have the package name, lets parse out the version.
found_package_version="$(echo "${found_package_filename}" | grep -oP '(?<=_)[^_]+(?=_)')"
display_alert "Found base-files upstream package version" "${found_package_version}" "info"
# Sanity check...
declare wanted_package_name="${sought_package_name}_${found_package_version}_${ARCH}.deb"
if [[ "${found_package_filename}" != "${wanted_package_name}" ]]; then
display_alert "Found package filename '${found_package_filename}' does not match wanted package name '${wanted_package_name}'" "looking for ${sought_package_name}" "warn"
return 1
fi
# show found_package_down_url
display_alert "Found package download url" "${found_package_down_url}" "debug"
return 0
}

View File

@@ -23,6 +23,10 @@ function determine_artifacts_to_build_for_image() {
fi
fi
# Userspace, RELEASE+ARCH specific, replaces the original distro's base-files
# This is always built, but only installed if KEEP_ORIGINAL_OS_RELEASE!=yes.
artifacts_to_build+=("armbian-base-files")
if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then
artifacts_to_build+=("fake_ubuntu_advantage_tools")
fi

View File

@@ -149,12 +149,7 @@ function install_distribution_agnostic() {
echo -e "${VENDOR} ${IMAGE_VERSION:-"${REVISION}"} ${RELEASE^} \\l \n" > "${SDCARD}"/etc/issue
echo "${VENDOR} ${IMAGE_VERSION:-"${REVISION}"} ${RELEASE^}" > "${SDCARD}"/etc/issue.net
# Keep, or change to Armbian's PRETTY_NAME in /etc/os-release (this is also done in the bsp-cli postinst)
if [[ "${KEEP_ORIGINAL_OS_RELEASE:-"no"}" != "yes" ]]; then
sed -i "s/^PRETTY_NAME=.*/PRETTY_NAME=\"${VENDOR} ${IMAGE_VERSION:-"${REVISION}"} ${RELEASE^}\"/" "${SDCARD}"/etc/os-release
else
display_alert "distro-agnostic: KEEP_ORIGINAL_OS_RELEASE" "Keeping original /etc/os-release's PRETTY_NAME as original" "info"
fi
# PRETTY_NAME changing in os-release is now done in armbian-base-files directly.
# enable few bash aliases enabled in Ubuntu by default to make it even
sed "s/#alias ll='ls -l'/alias ll='ls -l'/" -i "${SDCARD}"/etc/skel/.bashrc

View File

@@ -58,6 +58,11 @@ function install_distribution_specific() {
;;
esac
# install our base-files package (this replaces the original from Debian/Ubuntu)
if [[ "${KEEP_ORIGINAL_OS_RELEASE:-"no"}" != "yes" ]]; then
install_deb_chroot "${DEB_STORAGE}/${image_artifacts_debs["armbian-base-files"]}"
fi
# Basic Netplan config. Let NetworkManager/networkd manage all devices on this system
[[ -d "${SDCARD}"/etc/netplan ]] && cat <<- EOF > "${SDCARD}"/etc/netplan/armbian-default.yaml
network:

View File

@@ -1,6 +1,15 @@
#!/usr/bin/env bash
# This file is/was autogenerated by lib/tools/gen-library.sh; don't modify manually
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
set -o errtrace # trace ERR through - enabled
set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled
### lib/functions/artifacts/artifact-armbian-base-files.sh
# shellcheck source=lib/functions/artifacts/artifact-armbian-base-files.sh
source "${SRC}"/lib/functions/artifacts/artifact-armbian-base-files.sh
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
@@ -586,6 +595,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
# shellcheck source=lib/functions/configuration/package-lists.sh
source "${SRC}"/lib/functions/configuration/package-lists.sh
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
set -o errtrace # trace ERR through - enabled
set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled
### lib/functions/general/apt-utils.sh
# shellcheck source=lib/functions/general/apt-utils.sh
source "${SRC}"/lib/functions/general/apt-utils.sh
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled