diff --git a/lib/functions/cli/cli-configdump.sh b/lib/functions/cli/cli-configdump.sh index d99f256f1..15fec0d68 100644 --- a/lib/functions/cli/cli-configdump.sh +++ b/lib/functions/cli/cli-configdump.sh @@ -11,7 +11,6 @@ function cli_config_dump_run() { function config_and_remove_useless() { do_logging=no prepare_and_config_main_build_single # avoid logging during configdump; it's useless unset FINALDEST - unset FINAL_HOST_DEPS unset HOOK_ORDER HOOK_POINT HOOK_POINT_TOTAL_FUNCS unset REPO_CONFIG REPO_STORAGE unset DEB_STORAGE diff --git a/lib/functions/cli/cli-jsoninfo.sh b/lib/functions/cli/cli-jsoninfo.sh index 5be61de0e..3cc857d7f 100644 --- a/lib/functions/cli/cli-jsoninfo.sh +++ b/lib/functions/cli/cli-jsoninfo.sh @@ -11,9 +11,12 @@ function cli_json_info_run() { # - python debian packages hostdeps? (-dev, -pip, virtualenv, etc) # - run the virtualenv (messy?) + declare python3_binary_path + prepare_python3_binary_for_python_tools + # The info extractor itself... - run_host_command_logged python3 "${SRC}"/lib/tools/info.py ">" "${SRC}/output/info.json" + run_host_command_logged "${python3_binary_path}" "${SRC}"/lib/tools/info.py ">" "${SRC}/output/info.json" # Also convert output to CSV for easy import into Google Sheets etc - run_host_command_logged python3 "${SRC}"/lib/tools/json2csv.py "<" "${SRC}/output/info.json" ">" "${SRC}/output/info.csv" + run_host_command_logged "${python3_binary_path}" "${SRC}"/lib/tools/json2csv.py "<" "${SRC}/output/info.json" ">" "${SRC}/output/info.csv" } diff --git a/lib/functions/compilation/kernel-patching.sh b/lib/functions/compilation/kernel-patching.sh index 61ba100bb..a482d64ac 100644 --- a/lib/functions/compilation/kernel-patching.sh +++ b/lib/functions/compilation/kernel-patching.sh @@ -45,7 +45,9 @@ function kernel_main_patching_python() { "EXTRA_PATCH_HASHES_FIRST=${kernel_drivers_patch_hash}" # Is a space-separated list. ) display_alert "Calling Python patching script" "for kernel" "info" - run_host_command_logged env -i "${params_quoted[@]@Q}" python3 "${SRC}/lib/tools/patching.py" + declare python3_binary_path + prepare_python3_binary_for_python_tools + run_host_command_logged env -i "${params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/patching.py" run_host_command_logged cat "${temp_file_for_output}" # shellcheck disable=SC1090 source "${temp_file_for_output}" # SOURCE IT! diff --git a/lib/functions/compilation/uboot-patching.sh b/lib/functions/compilation/uboot-patching.sh index c5f27b4f0..b6435e5d8 100644 --- a/lib/functions/compilation/uboot-patching.sh +++ b/lib/functions/compilation/uboot-patching.sh @@ -30,7 +30,9 @@ function uboot_main_patching_python() { "BRANCH_FOR_PATCHES=u-boot-${BRANCH}-${BOARD}" # When applying patches-to-git, use this branch. ) display_alert "Calling Python patching script" "for u-boot target" "info" - run_host_command_logged env -i "${params_quoted[@]@Q}" python3 "${SRC}/lib/tools/patching.py" + declare python3_binary_path + prepare_python3_binary_for_python_tools + run_host_command_logged env -i "${params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/patching.py" run_host_command_logged cat "${temp_file_for_output}" # shellcheck disable=SC1090 source "${temp_file_for_output}" # SOURCE IT! diff --git a/lib/functions/configuration/aggregation.sh b/lib/functions/configuration/aggregation.sh index ab6848ae4..c28472299 100644 --- a/lib/functions/configuration/aggregation.sh +++ b/lib/functions/configuration/aggregation.sh @@ -56,7 +56,9 @@ function aggregate_all_packages() { "PACKAGE_LIST_BOARD_REMOVE=${PACKAGE_LIST_BOARD_REMOVE}" "PACKAGE_LIST_FAMILY_REMOVE=${PACKAGE_LIST_FAMILY_REMOVE}" ) - run_host_command_logged env -i "${aggregation_params_quoted[@]@Q}" python3 "${SRC}/lib/tools/aggregation.py" + declare python3_binary_path + prepare_python3_binary_for_python_tools + run_host_command_logged env -i "${aggregation_params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/aggregation.py" #run_host_command_logged cat "${temp_file_for_aggregation}" # shellcheck disable=SC1090 source "${temp_file_for_aggregation}" # SOURCE IT! diff --git a/lib/functions/general/python-tools.sh b/lib/functions/general/python-tools.sh index 1526849ed..78e33205c 100644 --- a/lib/functions/general/python-tools.sh +++ b/lib/functions/general/python-tools.sh @@ -23,3 +23,26 @@ function prepare_pip_packages_for_python_tools() { return 0 } + +# Called during early_prepare_host_dependencies(); when building a Dockerfile, HOSTRELEASE is set to the Docker image name. +function host_deps_add_extra_python() { + # Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9. + if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then + display_alert "Using Python 3.9 for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn" + host_dependencies+=("python3.9-dev" "python3.9-distutils") + else + display_alert "Using Python3 for" "'$HOSTRELEASE' has python3 >= 3.9" "debug" + fi +} + +# This sets the outer scope variable 'python3_binary_path' to /usr/bin/python3 or similar, depending on version. +function prepare_python3_binary_for_python_tools() { + python3_binary_path="/usr/bin/python3" + # Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9. + if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then + python3_binary_path="/usr/bin/python3.9" + display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn" + else + display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has python3 >= 3.9" "debug" + fi +} diff --git a/lib/functions/host/basic-deps.sh b/lib/functions/host/basic-deps.sh index cbeaadc49..1be5c4a62 100644 --- a/lib/functions/host/basic-deps.sh +++ b/lib/functions/host/basic-deps.sh @@ -3,7 +3,7 @@ # # * installs only basic packages # -prepare_host_basic() { +function prepare_host_basic() { # command:package1 package2 ... # list of commands that are neeeded:packages where this command is diff --git a/lib/functions/host/docker.sh b/lib/functions/host/docker.sh index 532cb00de..58c88f1c7 100755 --- a/lib/functions/host/docker.sh +++ b/lib/functions/host/docker.sh @@ -81,22 +81,23 @@ function docker_cli_prepare() { # @TODO: Make sure we can access docker, on Linux; gotta be part of 'docker' group: grep -q "$(whoami)" <(getent group docker) declare -g DOCKER_ARMBIAN_INITIAL_IMAGE_TAG="armbian.local.only/armbian-build:initial" - #declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:bookworm"}" # works Linux & Darwin - #declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:sid"}" # works Linux & Darwin - #declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:bullseye"}" # does NOT work under Darwin? loop problems. - #declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"ubuntu:kinetic"}" # works Linux & Darwin - declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"ubuntu:jammy"}" # works Linux & Darwin + # declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:bookworm"}" + # declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:sid"}" + # declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"debian:bullseye"}" + # declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"ubuntu:focal"}" + # declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"ubuntu:kinetic"}" + declare -g DOCKER_ARMBIAN_BASE_IMAGE="${DOCKER_ARMBIAN_BASE_IMAGE:-"ubuntu:jammy"}" declare -g DOCKER_ARMBIAN_TARGET_PATH="${DOCKER_ARMBIAN_TARGET_PATH:-"/armbian"}" + declare wanted_os_tag="${DOCKER_ARMBIAN_BASE_IMAGE%%:*}" + declare wanted_release_tag="${DOCKER_ARMBIAN_BASE_IMAGE##*:}" + # Store the "from scratch" image. Will be used if Armbian image is not available, for a "from scratch" build. declare -g DOCKER_ARMBIAN_BASE_IMAGE_SCRATCH="${DOCKER_ARMBIAN_BASE_IMAGE}" # If we're NOT building the public, official image, then USE the public, official image as base. # IMPORTANT: This has to match the naming scheme for tag the is used in the GitHub actions workflow. if [[ "${DOCKERFILE_USE_ARMBIAN_IMAGE_AS_BASE}" != "no" ]]; then - local wanted_os_tag="${DOCKER_ARMBIAN_BASE_IMAGE%%:*}" - local wanted_release_tag="${DOCKER_ARMBIAN_BASE_IMAGE##*:}" - # @TODO: this is rpardini's build. It's done in a different repo, so that's why the strange "armbian-release" name. It should be armbian/build:ubuntu-jammy-latest or something. DOCKER_ARMBIAN_BASE_IMAGE="ghcr.io/rpardini/armbian-release:armbian-next-${wanted_os_tag}-${wanted_release_tag}-latest" @@ -116,7 +117,7 @@ function docker_cli_prepare() { fi declare -a -g host_dependencies=() python3_pip_dependencies=() early_prepare_pip3_dependencies_for_python_tools - early_prepare_host_dependencies + HOSTRELEASE="${wanted_release_tag}" early_prepare_host_dependencies display_alert "Pre-game host dependencies" "${host_dependencies[*]}" "debug" display_alert "Pre-game pip3 dependencies" "${python3_pip_dependencies[*]}" "debug" diff --git a/lib/functions/host/host-release.sh b/lib/functions/host/host-release.sh new file mode 100644 index 000000000..3ffa7adbc --- /dev/null +++ b/lib/functions/host/host-release.sh @@ -0,0 +1,21 @@ +function obtain_and_check_hostrelease() { + # obtain the host release either from os-release or debian_version + declare -g HOSTRELEASE + HOSTRELEASE="$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2)" + [[ -z $HOSTRELEASE ]] && HOSTRELEASE="$(cut -d'/' -f1 /etc/debian_version)" + display_alert "Build host OS release" "${HOSTRELEASE:-(unknown)}" "info" + + # Ubuntu Jammy x86_64 or arm64 is the only fully supported host OS release + # Using Docker/VirtualBox is the only supported way to run the build script on other Linux distributions + # + # NO_HOST_RELEASE_CHECK overrides the check for a supported host system + # Disable host OS check at your own risk. Any issues reported with unsupported releases will be closed without discussion + if [[ -z $HOSTRELEASE || "bullseye bookworm sid focal impish hirsute jammy kinetic lunar ulyana ulyssa uma una vanessa vera" != *"$HOSTRELEASE"* ]]; then + if [[ $NO_HOST_RELEASE_CHECK == yes ]]; then + display_alert "You are running on an unsupported system" "${HOSTRELEASE:-(unknown)}" "wrn" + display_alert "Do not report any errors, warnings or other issues encountered beyond this point" "" "wrn" + else + exit_with_error "Unsupported build system: '${HOSTRELEASE:-(unknown)}'" + fi + fi +} diff --git a/lib/functions/host/prepare-host.sh b/lib/functions/host/prepare-host.sh index 68829d179..bd25d286a 100644 --- a/lib/functions/host/prepare-host.sh +++ b/lib/functions/host/prepare-host.sh @@ -63,25 +63,7 @@ prepare_host() { exit_with_error "Running this tool on non x86_64 or arm64 build host is not supported" fi - # obtain the host release either from os-release or debian_version - HOSTRELEASE=$(cat /etc/os-release | grep VERSION_CODENAME | cut -d"=" -f2) - [[ -z $HOSTRELEASE ]] && HOSTRELEASE=$(cut -d'/' -f1 /etc/debian_version) - - display_alert "Build host OS release" "${HOSTRELEASE:-(unknown)}" "info" - - # Ubuntu 21.04.x (Hirsute) x86_64 is the only fully supported host OS release - # Using Docker/VirtualBox/Vagrant is the only supported way to run the build script on other Linux distributions - # - # NO_HOST_RELEASE_CHECK overrides the check for a supported host system - # Disable host OS check at your own risk. Any issues reported with unsupported releases will be closed without discussion - if [[ -z $HOSTRELEASE || "buster bullseye bookworm focal impish hirsute jammy lunar kinetic debbie tricia ulyana ulyssa uma una vanessa vera" != *"$HOSTRELEASE"* ]]; then - if [[ $NO_HOST_RELEASE_CHECK == yes ]]; then - display_alert "You are running on an unsupported system" "${HOSTRELEASE:-(unknown)}" "wrn" - display_alert "Do not report any errors, warnings or other issues encountered beyond this point" "" "wrn" - else - exit_with_error "It seems you ignore documentation and run an unsupported build system: ${HOSTRELEASE:-(unknown)}" - fi - fi + obtain_and_check_hostrelease # sets HOSTRELEASE and validates it for sanity if grep -qE "(Microsoft|WSL)" /proc/version; then if [ -f /.dockerenv ]; then @@ -213,7 +195,9 @@ prepare_host() { function early_prepare_host_dependencies() { # packages list for host - # NOTE: please sync any changes here with the Dockerfile and Vagrantfile + # NOTE: those are automatically included in Dockerfile generation. + + # Common for all releases... declare -a -g host_dependencies=( # big bag of stuff from before acl aptly bc binfmt-support bison btrfs-progs @@ -227,27 +211,15 @@ function early_prepare_host_dependencies() { ntpdate patchutils pkg-config pv qemu-user-static rsync swig u-boot-tools udev uuid-dev whiptail - zlib1g-dev busybox fdisk + zlib1g-dev busybox - # distcc, experimental, optional; see cli-distcc.sh and kernel.sh - distcc - - # python3 stuff (eg, for modern u-boot) - python3-dev python3-distutils python3-setuptools - # python3 pip (for Armbian's Python utilities) @TODO virtualenv? - python3-pip - - # python2, including headers, mostly used by some u-boot builds (2017 et al, odroidxu4 and others). - python2 python2-dev - # Attention: 'python-setuptools' (Python2's setuptools) does not exist in Debian Sid. Use Python3 instead. - - # systemd-container brings in systemd-nspawn, which is used by the buildpkg functionality - # systemd-container # @TODO: bring this back eventually. I don't think trying to use those inside a container is a good idea. + distcc # distcc, experimental, optional; see cli-distcc.sh and kernel.sh # non-mess below? - file ccze colorized-logs tree expect # logging utilities; expect is needed for 'unbuffer' command + ccze colorized-logs # @TODO: drop those after removing usages + file tree expect # logging utilities; expect is needed for 'unbuffer' command unzip zip p7zip-full pigz pixz pbzip2 lzop zstd # compressors et al - parted gdisk # partition tools + parted gdisk fdisk # partition tools aria2 curl wget axel # downloaders et al parallel # do things in parallel # toolchains. NEW: using metapackages, allow us to have same list of all arches; brings both C and C++ compilers @@ -258,6 +230,24 @@ function early_prepare_host_dependencies() { libc6-amd64-cross # Support for running x86 binaries (under qemu on other arches) ) + # Now determine the HOSTRELEASE, if not passed-in. + if [[ "x${HOSTRELEASE}x" == "xx" ]]; then + obtain_and_check_hostrelease # Sets HOSTRELEASE & validates it for sanity + else + display_alert "Using passed-in HOSTRELEASE" "${HOSTRELEASE}" "warn" + fi + + host_deps_add_extra_python # Focal-like hosts might need extra python3.9 packages. + + # Python3 -- required for Armbian's Python tooling, and also for more recent u-boot builds. Needs 3.9+ + host_dependencies+=( + "python3-dev" "python3-distutils" "python3-setuptools" "python3-pip" + ) + + # Python2 -- required for some older u-boot builds + # Debian 'sid' does not carry python2 anymore. + host_dependencies+=("python2" "python2-dev") + # warning: apt-cacher-ng will fail if installed and used both on host and in container/chroot environment with shared network # set NO_APT_CACHER=yes to prevent installation errors in such case if [[ $NO_APT_CACHER != yes ]]; then @@ -298,7 +288,7 @@ function install_host_dependencies() { run_host_command_logged update-ccache-symlinks - export FINAL_HOST_DEPS="${host_dependencies[*]}" + declare -g FINAL_HOST_DEPS="${host_dependencies[*]}" call_extension_method "host_dependencies_ready" <<- 'HOST_DEPENDENCIES_READY' *run after all host dependencies are installed* @@ -306,4 +296,6 @@ function install_host_dependencies() { All the dependencies, including the default/core deps and the ones added via `${EXTRA_BUILD_DEPS}` are installed at this point. The system clock has not yet been synced. HOST_DEPENDENCIES_READY + + unset FINAL_HOST_DEPS # don't leak this after the hook is done } diff --git a/lib/library-functions.sh b/lib/library-functions.sh index 28446c6fc..a896ae007 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -487,6 +487,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/host/external-toolchains.sh source "${SRC}"/lib/functions/host/external-toolchains.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/host/host-release.sh +# shellcheck source=lib/functions/host/host-release.sh +source "${SRC}"/lib/functions/host/host-release.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled