mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
04f619dc06
Different build hosts have vastly different versions of setuptools and pyelftools depending on the host OS, e.g. Ubuntu 22.04 has setuptools v59 while the latest version at the time of this commit is setuptools v71. Using Pip instead of APT to download these packages assures that all build hosts use the same version, removing some points of failures and inconsistencies.
411 lines
18 KiB
Bash
411 lines
18 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
|
#
|
|
# This file is a part of the Armbian Build Framework
|
|
# https://github.com/armbian/build/
|
|
|
|
# prepare_host
|
|
#
|
|
# * checks and installs necessary packages
|
|
# * creates directory structure
|
|
# * changes system settings
|
|
#
|
|
function prepare_host() {
|
|
LOG_SECTION="prepare_host_noninteractive" do_with_logging prepare_host_noninteractive
|
|
return 0
|
|
}
|
|
|
|
function assert_prepared_host() {
|
|
if [[ "${PRE_PREPARED_HOST:-"no"}" == "yes" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ ${prepare_host_has_already_run:-0} -lt 1 ]]; then
|
|
exit_with_error "assert_prepared_host: Host has not yet been prepared. This is a bug in armbian-next code. Please report!"
|
|
fi
|
|
}
|
|
|
|
function check_basic_host() {
|
|
display_alert "Checking" "basic host setup" "info"
|
|
obtain_and_check_host_release_and_arch # sets HOSTRELEASE and validates it for sanity; also HOSTARCH
|
|
check_host_has_enough_disk_space # Checks disk space and exits if not enough
|
|
check_windows_wsl2 # checks if on Windows, on WSL2, (not 1) and exits if not supported
|
|
wait_for_package_manager # wait until dpkg is not locked...
|
|
}
|
|
|
|
function prepare_host_noninteractive() {
|
|
display_alert "Preparing" "host" "info"
|
|
|
|
# The 'offline' variable must always be set to 'true' or 'false'
|
|
declare offline=false
|
|
if [ "$OFFLINE_WORK" == "yes" ]; then
|
|
offline=true
|
|
fi
|
|
|
|
# fix for Locales settings, if locale-gen is installed, and /etc/locale.gen exists.
|
|
if [[ -n "$(command -v locale-gen)" && -f /etc/locale.gen ]]; then
|
|
if ! grep -q "^en_US.UTF-8 UTF-8" /etc/locale.gen; then
|
|
# @TODO: rpardini: this is bull, we're always root here. we've been pre-sudo'd.
|
|
local sudo_prefix="" && is_root_or_sudo_prefix sudo_prefix # nameref; "sudo_prefix" will be 'sudo' or ''
|
|
${sudo_prefix} sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
|
|
${sudo_prefix} locale-gen
|
|
fi
|
|
else
|
|
display_alert "locale-gen is not installed @host" "skipping locale-gen -- problems might arise" "warn"
|
|
fi
|
|
|
|
# Let's try and get all log output in English, overriding the builder's chosen or default language
|
|
export LANG="en_US.UTF-8"
|
|
export LANGUAGE="en_US.UTF-8"
|
|
export LC_ALL="en_US.UTF-8"
|
|
export LC_MESSAGES="en_US.UTF-8"
|
|
|
|
declare -g USE_LOCAL_APT_DEB_CACHE=${USE_LOCAL_APT_DEB_CACHE:-yes} # Use SRC/cache/aptcache as local apt cache by default
|
|
display_alert "Using local apt cache?" "USE_LOCAL_APT_DEB_CACHE: ${USE_LOCAL_APT_DEB_CACHE}" "debug"
|
|
|
|
# if USE_LOCAL_APT_DEB_CACHE equals no, display_alert a warning, it's not a good idea.
|
|
if [[ "${USE_LOCAL_APT_DEB_CACHE}" == "no" ]]; then
|
|
display_alert "USE_LOCAL_APT_DEB_CACHE is set to 'no'" "not recommended" "wrn"
|
|
fi
|
|
|
|
if armbian_is_running_in_container; then
|
|
display_alert "Running in container" "Adding provisions for container building" "info"
|
|
declare -g CONTAINER_COMPAT=yes # this controls mknod usage for loop devices.
|
|
|
|
if [[ "${MANAGE_ACNG}" == "yes" ]]; then
|
|
display_alert "Running in container" "Disabling ACNG - MANAGE_ACNG=yes not supported in containers" "warn"
|
|
declare -g MANAGE_ACNG=no
|
|
fi
|
|
|
|
SYNC_CLOCK=no
|
|
else
|
|
display_alert "NOT running in container" "No special provisions for container building" "debug"
|
|
fi
|
|
|
|
# If offline, do not try to install dependencies, manage acng, or sync the clock.
|
|
if ! $offline; then
|
|
# Prepare the list of host dependencies; it requires the target arch, the host release and arch
|
|
late_prepare_host_dependencies
|
|
install_host_dependencies "late dependencies during prepare_release"
|
|
|
|
# Manage apt-cacher-ng, if such is the case
|
|
[[ "${MANAGE_ACNG}" == "yes" ]] && acng_configure_and_restart_acng
|
|
|
|
# sync clock
|
|
if [[ $SYNC_CLOCK != no && -f /var/run/ntpd.pid ]]; then
|
|
display_alert "ntpd is running, skipping" "SYNC_CLOCK=no" "debug"
|
|
SYNC_CLOCK=no
|
|
fi
|
|
|
|
if [[ $SYNC_CLOCK != no ]]; then
|
|
display_alert "Syncing clock" "host" "info"
|
|
run_host_command_logged ntpdate "${NTP_SERVER:-pool.ntp.org}" || true # allow failures
|
|
fi
|
|
fi
|
|
|
|
# create directory structure # @TODO: this should be close to DEST, otherwise super-confusing
|
|
mkdir -p "${SRC}"/{cache,output} "${USERPATCHES_PATH}" "${SRC}"/output/info
|
|
|
|
# @TODO: original: mkdir -p "${DEST}"/debs-beta/extra "${DEST}"/debs/extra "${DEST}"/{config,debug,patch} "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,hash,hash-beta,toolchain,utility,rootfs} "${SRC}"/.tmp
|
|
mkdir -p "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,rootfs} "${SRC}"/.tmp
|
|
|
|
# If offline, do not try to download/install toolchains.
|
|
if ! $offline; then
|
|
download_external_toolchains # Mostly deprecated, since SKIP_EXTERNAL_TOOLCHAINS=yes is the default
|
|
fi
|
|
|
|
# NEEDS_BINFMT=yes is set by default build and rootfs artifact build.
|
|
# if we're building an image, not only packages/artifacts...
|
|
# ... and the host arch does not match the target arch ...
|
|
# ... we then require binfmt_misc to be enabled.
|
|
# "enable arm binary format so that the cross-architecture chroot environment will work"
|
|
if [[ "${NEEDS_BINFMT:-"no"}" == "yes" ]]; then
|
|
|
|
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
|
|
display_alert "Debugging binfmt - early" "/proc/sys/fs/binfmt_misc/" "debug"
|
|
run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
|
|
fi
|
|
|
|
if dpkg-architecture -e "${ARCH}"; then
|
|
display_alert "Native arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "cachehit"
|
|
else
|
|
local failed_binfmt_modprobe=0
|
|
|
|
display_alert "Cross arch build" "target ${ARCH} on host $(dpkg --print-architecture)" "debug"
|
|
|
|
# Check if binfmt_misc is loaded; if not, try to load it, but don't fail: it might be built in.
|
|
if grep -q "^binfmt_misc" /proc/modules; then
|
|
display_alert "binfmt_misc is already loaded" "binfmt_misc already loaded" "debug"
|
|
else
|
|
display_alert "binfmt_misc is not loaded" "trying to load binfmt_misc" "debug"
|
|
|
|
# try to modprobe. if it fails, emit a warning later, but not here.
|
|
# this is for the in-container case, where the host already has the module, but won't let the container know about it.
|
|
modprobe -q binfmt_misc || failed_binfmt_modprobe=1
|
|
fi
|
|
|
|
# Now, /proc/sys/fs/binfmt_misc/ has to be mounted. Mount, or fail with a message
|
|
if mountpoint -q /proc/sys/fs/binfmt_misc/; then
|
|
display_alert "binfmt_misc is already mounted" "binfmt_misc already mounted" "debug"
|
|
else
|
|
display_alert "binfmt_misc is not mounted" "trying to mount binfmt_misc" "debug"
|
|
mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc/ || {
|
|
if [[ $failed_binfmt_modprobe == 1 ]]; then
|
|
display_alert "Failed to load binfmt_misc module" "modprobe binfmt_misc failed" "wrn"
|
|
fi
|
|
display_alert "Check your HOST kernel" "CONFIG_BINFMT_MISC=m is required in host kernel" "warn"
|
|
display_alert "Failed to mount" "binfmt_misc /proc/sys/fs/binfmt_misc/" "err"
|
|
exit_with_error "Failed to mount binfmt_misc"
|
|
}
|
|
display_alert "binfmt_misc mounted" "binfmt_misc mounted" "debug"
|
|
fi
|
|
|
|
declare host_arch
|
|
host_arch="$(arch)"
|
|
local -a wanted_arches=("arm" "aarch64" "x86_64" "riscv64")
|
|
display_alert "Preparing binfmts for arch" "binfmts: host '${host_arch}', wanted arches '${wanted_arches[*]}'" "debug"
|
|
declare wanted_arch
|
|
for wanted_arch in "${wanted_arches[@]}"; do
|
|
if [[ "${host_arch}" != "${wanted_arch}" ]]; then
|
|
if [[ ! -e "/proc/sys/fs/binfmt_misc/qemu-${wanted_arch}" ]]; then
|
|
display_alert "Updating binfmts" "update-binfmts --enable qemu-${wanted_arch}" "debug"
|
|
if [[ "${host_arch}" == "aarch64" && "${wanted_arch}" == "arm" ]]; then
|
|
display_alert "Trying to update binfmts - aarch64 (sometimes) does 32-bit sans emulation" "update-binfmts --enable qemu-${wanted_arch}" "debug"
|
|
run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" "&>" "/dev/null" "||" "true" # don't fail nor produce output, which can be misleading.
|
|
else
|
|
run_host_command_logged update-binfmts --enable "qemu-${wanted_arch}" || display_alert "Failed to update binfmts" "update-binfmts --enable qemu-${wanted_arch}" "err" # log & continue on failure
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# @TODO: we could create a tiny loop here to test if the binfmt_misc is working, but this is before deps are installed.
|
|
fi
|
|
|
|
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
|
|
display_alert "Debugging binfmt - late" "/proc/sys/fs/binfmt_misc/" "debug"
|
|
run_host_command_logged ls -la /proc/sys/fs/binfmt_misc/ || true
|
|
fi
|
|
|
|
fi
|
|
|
|
# @TODO: rpardini: this does not belong here, instead with the other templates, pre-configuration.
|
|
[[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && run_host_command_logged cp -pv "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh
|
|
|
|
if [[ -d "${USERPATCHES_PATH}" ]]; then
|
|
# create patches directory structure under USERPATCHES_PATH
|
|
find "${SRC}"/patch -maxdepth 2 -type d ! -name . | sed "s%/.*patch%/$USERPATCHES_PATH%" | xargs mkdir -p
|
|
fi
|
|
|
|
# Reset owner of userpatches if so required
|
|
reset_uid_owner "${USERPATCHES_PATH}" # Fix owner of files in the final destination
|
|
|
|
declare -i -g -r prepare_host_has_already_run=1 # global, readonly.
|
|
|
|
return 0
|
|
}
|
|
|
|
# Early: we've possibly no idea what the host release or arch we're building on, or what the target arch is. All-deps.
|
|
# Early: we've a best-guess indication of the host release, but not target. (eg: Dockerfile generate)
|
|
# Early: we're certain about the host release and arch, but not anything about the target (eg: docker build of the Dockerfile, cli-requirements)
|
|
# Late: we know everything; produce a list that is optimized for the host+target we're building. (eg: Oleg)
|
|
function early_prepare_host_dependencies() {
|
|
if [[ "x${host_release:-}x" == "xx" ]]; then
|
|
display_alert "Host release unknown" "host_release not set on call to early_prepare_host_dependencies" "warn"
|
|
fi
|
|
if [[ "x${host_arch:-}x" == "xx" ]]; then
|
|
display_alert "Host arch unknown" "host_arch not set on call to early_prepare_host_dependencies" "debug"
|
|
fi
|
|
adaptative_prepare_host_dependencies
|
|
}
|
|
|
|
function late_prepare_host_dependencies() {
|
|
[[ -z "${ARCH}" ]] && exit_with_error "ARCH is not set"
|
|
[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"
|
|
[[ -z "${HOSTARCH}" ]] && exit_with_error "HOSTARCH is not set"
|
|
[[ -z "${RELEASE}" ]] && display_alert "RELEASE is not set" "defaulting to host's '${HOSTRELEASE}'" "debug"
|
|
|
|
target_arch="${ARCH}" host_release="${HOSTRELEASE}" \
|
|
host_arch="${HOSTARCH}" target_release="${RELEASE:-"${HOSTRELEASE}"}" \
|
|
early_prepare_host_dependencies
|
|
}
|
|
|
|
# Adaptive: used by both early & late.
|
|
function adaptative_prepare_host_dependencies() {
|
|
if [[ "x${host_release:-"unknown"}x" == "xx" ]]; then
|
|
display_alert "No specified host_release" "preparing for all-hosts, all-targets deps" "debug"
|
|
else
|
|
display_alert "Using passed-in host_release" "${host_release}" "debug"
|
|
fi
|
|
|
|
if [[ "x${target_arch:-"unknown"}x" == "xx" ]]; then
|
|
display_alert "No specified target_arch" "preparing for all-hosts, all-targets deps" "debug"
|
|
else
|
|
display_alert "Using passed-in target_arch" "${target_arch}" "debug"
|
|
fi
|
|
|
|
#### Common: for all releases, all host arches, and all target arches.
|
|
declare -a -g host_dependencies=(
|
|
# big bag of stuff from before
|
|
bc binfmt-support
|
|
bison
|
|
libc6-dev make dpkg-dev gcc # build-essential, without g++
|
|
ca-certificates ccache cpio
|
|
device-tree-compiler dialog dirmngr dosfstools
|
|
dwarves # dwarves has been replaced by "pahole" and is now a transitional package
|
|
flex
|
|
gawk gnupg gpg
|
|
imagemagick # required for plymouth: converting images / spinners
|
|
jq # required for parsing JSON, specially rootfs-caching related.
|
|
kmod # this causes initramfs rebuild, but is usually pre-installed, so no harm done unless it's an upgrade
|
|
libbison-dev libelf-dev libfdt-dev libfile-fcntllock-perl libmpc-dev libfl-dev lz4
|
|
libncurses-dev libssl-dev libusb-1.0-0-dev
|
|
linux-base locales lsof
|
|
ncurses-base ncurses-term # for `make menuconfig`
|
|
ntpdate
|
|
patchutils pkg-config pv
|
|
qemu-user-static
|
|
rsync
|
|
swig # swig is needed for some u-boot's. example: "bananapi.conf"
|
|
u-boot-tools
|
|
udev # causes initramfs rebuild, but is usually pre-installed.
|
|
uuid-dev
|
|
zlib1g-dev
|
|
|
|
# by-category below
|
|
file tree expect # logging utilities; expect is needed for 'unbuffer' command
|
|
colorized-logs # for ansi2html, ansi2txt, pipetty
|
|
unzip zip pigz xz-utils pbzip2 lzop zstd # compressors et al
|
|
parted gdisk fdisk # partition tools @TODO why so many?
|
|
aria2 curl wget axel # downloaders et al
|
|
parallel # do things in parallel (used for fast md5 hashing in initrd cache)
|
|
)
|
|
|
|
# @TODO: distcc -- handle in extension?
|
|
|
|
### Python
|
|
host_deps_add_extra_python # See python-tools.sh::host_deps_add_extra_python()
|
|
|
|
### Python3 -- required for Armbian's Python tooling, and also for more recent u-boot builds. Needs 3.9+; ffi-dev is needed for some Python packages when the wheel is not prebuilt
|
|
### 'python3-setuptools' and 'python3-pyelftools' moved to requirements.txt to make sure build hosts use the same/latest versions of these tools.
|
|
host_dependencies+=("python3-dev" "python3-pip" "libffi-dev")
|
|
|
|
# Needed for some u-boot's, lest "tools/mkeficapsule.c:21:10: fatal error: gnutls/gnutls.h"
|
|
host_dependencies+=("libgnutls28-dev")
|
|
|
|
# Noble and later releases do not carry "python3-distutils" https://docs.python.org/3.10/whatsnew/3.10.html#distutils-deprecated
|
|
if [[ "noble" == *"${host_release}"* ]]; then
|
|
display_alert "python3-distutils not available on host release '${host_release}'" "distutils was deprecated with Python 3.12" "debug"
|
|
else
|
|
host_dependencies+=("python3-distutils")
|
|
fi
|
|
|
|
### Python2 -- required for some older u-boot builds
|
|
# Debian 'sid'/'bookworm' and Ubuntu 'lunar' does not carry python2 anymore; in this case some u-boot's might fail to build.
|
|
if [[ "sid bookworm trixie lunar mantic noble" == *"${host_release}"* ]]; then
|
|
display_alert "Python2 not available on host release '${host_release}'" "old(er) u-boot builds might/will fail" "wrn"
|
|
else
|
|
host_dependencies+=("python2" "python2-dev")
|
|
fi
|
|
|
|
# Only install acng if asked to.
|
|
if [[ "${MANAGE_ACNG}" == "yes" ]]; then
|
|
host_dependencies+=("apt-cacher-ng")
|
|
fi
|
|
|
|
### ARCH
|
|
declare wanted_arch="${target_arch:-"all"}"
|
|
|
|
if [[ "${wanted_arch}" == "amd64" || "${wanted_arch}" == "all" ]]; then
|
|
host_dependencies+=("gcc-x86-64-linux-gnu") # from crossbuild-essential-amd64
|
|
fi
|
|
|
|
if [[ "${wanted_arch}" == "arm64" || "${wanted_arch}" == "all" ]]; then
|
|
host_dependencies+=("gcc-aarch64-linux-gnu") # from crossbuild-essential-arm64
|
|
fi
|
|
|
|
if [[ "${wanted_arch}" == "armhf" || "${wanted_arch}" == "all" ]]; then
|
|
host_dependencies+=("gcc-arm-linux-gnueabihf" "gcc-arm-linux-gnueabi") # from crossbuild-essential-armhf crossbuild-essential-armel
|
|
fi
|
|
|
|
if [[ "${wanted_arch}" == "riscv64" || "${wanted_arch}" == "all" ]]; then
|
|
host_dependencies+=("gcc-riscv64-linux-gnu") # crossbuild-essential-riscv64 is not even available "yet"
|
|
host_dependencies+=("debian-archive-keyring")
|
|
fi
|
|
|
|
if [[ "${wanted_arch}" != "amd64" ]]; then
|
|
host_dependencies+=(libc6-amd64-cross) # Support for running x86 binaries (under qemu on other arches)
|
|
fi
|
|
|
|
declare -g EXTRA_BUILD_DEPS=""
|
|
call_extension_method "add_host_dependencies" <<- 'ADD_HOST_DEPENDENCIES'
|
|
*run before installing host dependencies*
|
|
you can add packages to install, space separated, to ${EXTRA_BUILD_DEPS} here.
|
|
ADD_HOST_DEPENDENCIES
|
|
|
|
if [ -n "${EXTRA_BUILD_DEPS}" ]; then
|
|
# shellcheck disable=SC2206 # I wanna expand. @TODO: later will convert to proper array
|
|
host_dependencies+=(${EXTRA_BUILD_DEPS})
|
|
fi
|
|
|
|
declare -g FINAL_HOST_DEPS="${host_dependencies[*]}"
|
|
call_extension_method "host_dependencies_known" <<- 'HOST_DEPENDENCIES_KNOWN'
|
|
*run after all host dependencies are known (but not installed)*
|
|
At this point we can read `${FINAL_HOST_DEPS}`, but changing won't have any effect.
|
|
All the dependencies, including the default/core deps and the ones added via `${EXTRA_BUILD_DEPS}`
|
|
are determined at this point, but not yet installed.
|
|
HOST_DEPENDENCIES_KNOWN
|
|
}
|
|
|
|
function install_host_dependencies() {
|
|
display_alert "Installing build dependencies" "$*" "debug"
|
|
|
|
# don't prompt for apt cacher selection. this is to skip the prompt only, since we'll manage acng config later.
|
|
local sudo_prefix="" && is_root_or_sudo_prefix sudo_prefix # nameref; "sudo_prefix" will be 'sudo' or ''
|
|
${sudo_prefix} echo "apt-cacher-ng apt-cacher-ng/tunnelenable boolean false" | ${sudo_prefix} debconf-set-selections
|
|
|
|
# This handles the wanted list in $host_dependencies, updates apt only if needed
|
|
# $host_dependencies is produced by early_prepare_host_dependencies()
|
|
install_host_side_packages "${host_dependencies[@]}"
|
|
|
|
run_host_command_logged update-ccache-symlinks
|
|
|
|
declare -g FINAL_HOST_DEPS="${host_dependencies[*]}"
|
|
|
|
call_extension_method "host_dependencies_ready" <<- 'HOST_DEPENDENCIES_READY'
|
|
*run after all host dependencies are installed*
|
|
At this point we can read `${FINAL_HOST_DEPS}`, but changing won't have any effect.
|
|
All the dependencies, including the default/core deps and the ones added via `${EXTRA_BUILD_DEPS}`
|
|
are installed at this point. The system clock has not yet been synced.
|
|
HOST_DEPENDENCIES_READY
|
|
|
|
unset FINAL_HOST_DEPS # don't leak this after the hook is done
|
|
}
|
|
|
|
function check_host_has_enough_disk_space() {
|
|
declare -a dirs_to_check=("${DEST}" "${SRC}/cache")
|
|
for dir in "${dirs_to_check[@]}"; do
|
|
if [[ ! -d "${dir}" ]]; then
|
|
display_alert "Directory not found" "Skipping disk space check for '${dir}'" "debug"
|
|
continue
|
|
fi
|
|
check_dir_has_enough_disk_space "${dir}" 10 || exit_if_countdown_not_aborted 10 "Low free disk space left in '${dir}'"
|
|
done
|
|
}
|
|
|
|
function check_dir_has_enough_disk_space() {
|
|
declare target="${1}"
|
|
declare -i min_free_space_gib="${2:-10}"
|
|
declare -i free_space_bytes
|
|
free_space_bytes=$(findmnt --noheadings --output AVAIL --bytes --target "${target}" --uniq 2> /dev/null) # in bytes
|
|
if [[ -n "$free_space_bytes" && $((free_space_bytes / 1073741824)) -lt $min_free_space_gib ]]; then
|
|
display_alert "Low free space left" "${target}: $((free_space_bytes / 1073741824))GiB free, ${min_free_space_gib} GiB required" "wrn"
|
|
return 1
|
|
fi
|
|
display_alert "Free space left" "${target}: $((free_space_bytes / 1073741824))GiB" "debug"
|
|
return 0
|
|
}
|