diff --git a/lib/functions/cli/cli-patch.sh b/lib/functions/cli/cli-patch.sh index 2cf57b38e..e461a43d4 100644 --- a/lib/functions/cli/cli-patch.sh +++ b/lib/functions/cli/cli-patch.sh @@ -34,7 +34,7 @@ function cli_patch_kernel_run() { "kernel-${LINUXFAMILY}-${KERNEL_MAJOR_MINOR}:${target_branch}") # Prepare the host and build kernel instead of main_default_build_single - LOG_SECTION="prepare_host" do_with_logging prepare_host + prepare_host # This handles its own logging sections, and is possibly interactive. compile_kernel # This handles its own logging sections. display_alert "Done patching kernel" "${BRANCH} - ${LINUXFAMILY} - ${KERNEL_MAJOR_MINOR}" "cachehit" diff --git a/lib/functions/general/cleaning.sh b/lib/functions/general/cleaning.sh index 2a812ebe5..2b2ae6fc9 100644 --- a/lib/functions/general/cleaning.sh +++ b/lib/functions/general/cleaning.sh @@ -86,3 +86,21 @@ function general_cleaning() { return 0 # a LOT of shortcircuits above; prevent spurious error messages } + +function clean_deprecated_mountpoints() { + # Cleaning of old, deprecated mountpoints; only done if not running under Docker. + # mountpoints under Docker manifest as volumes, and as such can't be cleaned this way. + if [[ "${ARMBIAN_RUNNING_IN_CONTAINER}" != "yes" ]]; then + prepare_armbian_mountpoints_description_dict + local mountpoint="" + for mountpoint in "${ARMBIAN_MOUNTPOINTS_DEPRECATED[@]}"; do + local mountpoint_dir="${SRC}/${mountpoint}" + display_alert "Considering cleaning deprecated mountpoint" "${mountpoint_dir}" "debug" + if [[ -d "${mountpoint_dir}" ]]; then + display_alert "Cleaning deprecated mountpoint" "${mountpoint_dir}" "info" + run_host_command_logged rm -rf "${mountpoint_dir}" + fi + done + fi + return 0 +} diff --git a/lib/functions/host/host-utils.sh b/lib/functions/host/host-utils.sh index 26b1ec5af..56e41ae25 100644 --- a/lib/functions/host/host-utils.sh +++ b/lib/functions/host/host-utils.sh @@ -204,7 +204,7 @@ function mkdir_recursive_and_set_uid_owner() { done } -# Call: reset_uid_owner "one/file" "some/directory" "another/file" +# Call: reset_uid_owner "one/file" "some/directory" "another/file" - is recursive if dir given function reset_uid_owner() { if [[ "x${SET_OWNER_TO_UID}x" == "xx" ]]; then return 0 # Nothing to do. @@ -225,6 +225,26 @@ function reset_uid_owner() { done } +# Non recursive version of the above +function reset_uid_owner_non_recursive() { + if [[ "x${SET_OWNER_TO_UID}x" == "xx" ]]; then + return 0 # Nothing to do. + fi + # Loop over args.. + local arg + for arg in "$@"; do + display_alert "reset_uid_owner_non_recursive: '${arg}' will be owner id '${SET_OWNER_TO_UID}'" "reset_uid_owner_non_recursive" "debug" + if [[ -d "${arg}" ]]; then + chown "${SET_OWNER_TO_UID}" "${arg}" + elif [[ -f "${arg}" ]]; then + chown "${SET_OWNER_TO_UID}" "${arg}" + else + display_alert "reset_uid_owner_non_recursive: '${arg}' is not a file or directory" "skipping" "debug" + return 1 + fi + done +} + # call: check_dir_for_mount_options "/path/to/dir" "main build dir description" function check_dir_for_mount_options() { declare -r dir="${1}" @@ -248,6 +268,7 @@ function check_dir_for_mount_options() { function trap_handler_reset_output_owner() { display_alert "Resetting output directory owner" "${SRC}/output" "debug" reset_uid_owner "${SRC}/output" + # For .tmp: do NOT do it recursively. If another build is running in another process, this is destructive if recursive. display_alert "Resetting tmp directory owner" "${SRC}/.tmp" "debug" - reset_uid_owner "${SRC}/.tmp" + reset_uid_owner_non_recursive "${SRC}/.tmp" } diff --git a/lib/functions/host/prepare-host.sh b/lib/functions/host/prepare-host.sh index 468fb28ec..b3ba7d1b1 100644 --- a/lib/functions/host/prepare-host.sh +++ b/lib/functions/host/prepare-host.sh @@ -5,19 +5,26 @@ # * creates directory structure # * changes system settings # -prepare_host() { +function prepare_host() { + # Those are not logged, and might be interactive. + display_alert "Checking" "host" "info" + obtain_and_check_host_release_and_arch # sets HOSTRELEASE and validates it for sanity; also HOSTARCH + check_host_has_enough_disk_space # Checks disk space and exits if not enough + wait_for_package_manager # wait until dpkg is not locked... + + LOG_SECTION="prepare_host_noninteractive" do_with_logging prepare_host_noninteractive + return 0 +} + +function prepare_host_noninteractive() { display_alert "Preparing" "host" "info" # The 'offline' variable must always be set to 'true' or 'false' + declare offline=false if [ "$OFFLINE_WORK" == "yes" ]; then - local offline=true - else - local offline=false + offline=true fi - # wait until package manager finishes possible system maintanace - wait_for_package_manager - # fix for Locales settings, if locale-gen is installed, and /etc/locale.gen exists. if [[ -n "$(command -v locale-gen)" && -f /etc/locale.gen ]]; then if ! grep -q "^en_US.UTF-8 UTF-8" /etc/locale.gen; then @@ -53,16 +60,6 @@ prepare_host() { done fi - obtain_and_check_host_release_and_arch # sets HOSTRELEASE and validates it for sanity; also HOSTARCH - - if grep -qE "(Microsoft|WSL)" /proc/version; then - if [ -f /.dockerenv ]; then - display_alert "Building images using Docker on WSL2 may fail" "" "wrn" - else - exit_with_error "Windows subsystem for Linux is not a supported build environment" - fi - fi - declare -g USE_LOCAL_APT_DEB_CACHE=${USE_LOCAL_APT_DEB_CACHE:-yes} # Use SRC/cache/aptcache as local apt cache by default display_alert "Using local apt cache?" "USE_LOCAL_APT_DEB_CACHE: ${USE_LOCAL_APT_DEB_CACHE}" "debug" @@ -111,12 +108,11 @@ prepare_host() { mkdir -p "${SRC}"/{cache,output} "${USERPATCHES_PATH}" # @TODO: original: mkdir -p "${DEST}"/debs-beta/extra "${DEST}"/debs/extra "${DEST}"/{config,debug,patch} "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,hash,hash-beta,toolchain,utility,rootfs} "${SRC}"/.tmp - mkdir -p "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,hash,hash-beta,toolchain,utility,rootfs} "${SRC}"/.tmp + mkdir -p "${USERPATCHES_PATH}"/overlay "${SRC}"/cache/{sources,rootfs} "${SRC}"/.tmp # If offline, do not try to download/install toolchains. if ! $offline; then - # Mostly deprecated. - download_external_toolchains + download_external_toolchains # Mostly deprecated, since SKIP_EXTERNAL_TOOLCHAINS=yes is the default fi # if we're building an image, not only packages... @@ -163,18 +159,15 @@ prepare_host() { test -e /proc/sys/fs/binfmt_misc/qemu-arm || update-binfmts --enable qemu-arm test -e /proc/sys/fs/binfmt_misc/qemu-aarch64 || update-binfmts --enable qemu-aarch64 fi + + # @TODO: we could create a tiny loop here to test if the binfmt_misc is working, but this is before deps are installed. fi fi # @TODO: rpardini: this does not belong here, instead with the other templates, pre-configuration. [[ ! -f "${USERPATCHES_PATH}"/customize-image.sh ]] && run_host_command_logged cp -pv "${SRC}"/config/templates/customize-image.sh.template "${USERPATCHES_PATH}"/customize-image.sh - # @TODO: what is this, and why? - if [[ ! -f "${USERPATCHES_PATH}"/README ]]; then - rm -f "${USERPATCHES_PATH}"/readme.txt - echo 'Please read documentation about customizing build configuration' > "${USERPATCHES_PATH}"/README - echo 'https://www.armbian.com/using-armbian-tools/' >> "${USERPATCHES_PATH}"/README - + if [[ -d "${USERPATCHES_PATH}" ]]; then # create patches directory structure under USERPATCHES_PATH find "${SRC}"/patch -maxdepth 2 -type d ! -name . | sed "s%/.*patch%/$USERPATCHES_PATH%" | xargs mkdir -p fi @@ -182,16 +175,7 @@ prepare_host() { # Reset owner of userpatches if so required reset_uid_owner "${USERPATCHES_PATH}" # Fix owner of files in the final destination - # @TODO: check every possible mount point. Not only one. People might have different mounts / Docker volumes... - # check free space (basic) @TODO probably useful to refactor and implement in multiple spots. - declare -i free_space_bytes - free_space_bytes=$(findmnt --noheadings --output AVAIL --bytes --target "${SRC}" --uniq 2> /dev/null) # in bytes - if [[ -n "$free_space_bytes" && $((free_space_bytes / 1073741824)) -lt 10 ]]; then - display_alert "Low free space left" "$((free_space_bytes / 1073741824)) GiB" "wrn" - # pause here since dialog-based menu will hide this message otherwise - echo -e "Press \e[0;33m\x1B[0m to abort compilation, \e[0;33m\x1B[0m to ignore and continue" - read # @TODO: this fails if stdin is not a tty, or just hangs - fi + return 0 } # Early: we've possibly no idea what the host release or arch we're building on, or what the target arch is. All-deps. @@ -356,3 +340,14 @@ function install_host_dependencies() { unset FINAL_HOST_DEPS # don't leak this after the hook is done } + +function check_host_has_enough_disk_space() { + # @TODO: check every possible mount point. Not only one. People might have different mounts / Docker volumes... + # check free space (basic) @TODO probably useful to refactor and implement in multiple spots. + declare -i free_space_bytes + free_space_bytes=$(findmnt --noheadings --output AVAIL --bytes --target "${SRC}" --uniq 2> /dev/null) # in bytes + if [[ -n "$free_space_bytes" && $((free_space_bytes / 1073741824)) -lt 10 ]]; then + display_alert "Low free space left" "$((free_space_bytes / 1073741824))GiB" "wrn" + exit_if_countdown_not_aborted 10 "Low free disk space left" # This pauses & exits if error if ENTER is not pressed in 10 seconds + fi +} diff --git a/lib/functions/logging/traps.sh b/lib/functions/logging/traps.sh index 5f52850a1..dbf768788 100644 --- a/lib/functions/logging/traps.sh +++ b/lib/functions/logging/traps.sh @@ -165,3 +165,35 @@ function exit_with_error() { exit 43 } + +# This exits, unless user presses ENTER. If not interactive, user will be unable to comply and the script will exit. +function exit_if_countdown_not_aborted() { + # parse + declare -i loops="${1}" + declare reason="${2}" + # validate + [[ -z "${loops}" ]] && exit_with_error "countdown_to_exit_or_just_exit_if_noninteractive() called without a number of loops" + [[ -z "${reason}" ]] && exit_with_error "countdown_to_exit_or_just_exit_if_noninteractive() called without a reason" + + # If not interactive, just exit. + if [[ ! -t 1 ]]; then + exit_with_error "Exiting due to '${reason}' - not interactive, exiting immediately." + fi + + display_alert "Problem detected" "${reason}" "err" + display_alert "Exiting in ${loops} seconds" "Press \e[0;33m\x1B[0m to abort, \e[0;33m\x1B[0m to ignore and continue" "err" + echo -n "Counting down: " + for i in $(seq 1 "${loops}"); do + declare stop_waiting=0 + declare keep_waiting=0 + timeout --foreground 1 bash -c "read -n1; echo \$REPLY" && stop_waiting=1 || keep_waiting=1 + if [[ "$stop_waiting" == "1" ]]; then + display_alert "User pressed ENTER, continuing, albeit" "${reason}" "wrn" + return 0 + fi + echo -n "$i... " >&2 + done + + # Countdown finished, exit. + exit_with_error "Exiting due to '${reason}'" +} diff --git a/lib/functions/main/default-build.sh b/lib/functions/main/default-build.sh index 8a6ea0065..e477629b4 100644 --- a/lib/functions/main/default-build.sh +++ b/lib/functions/main/default-build.sh @@ -16,18 +16,18 @@ function main_default_build_single() { LOG_SECTION="prepare_tmpfs_workdir" do_with_logging prepare_tmpfs_for "WORKDIR" "${WORKDIR}" # this adds its own cleanup handler, which deletes it if it was created add_cleanup_handler trap_handler_cleanup_workdir # this is for when it is NOT a tmpfs, for any reason; it does not delete the dir itself. - # '-x': export + # 'declare -g -x': global, export declare -g -x TMPDIR="${WORKDIR}" # TMPDIR is default for a lot of stuff, but... declare -g -x CCACHE_TEMPDIR="${WORKDIR}/ccache_tmp" # Export CCACHE_TEMPDIR, under Workdir, which is hopefully under tmpfs. Thanks @the-Going for this. declare -g -x XDG_RUNTIME_DIR="${WORKDIR}/xdg_tmp" # XDG_RUNTIME_DIR is used by the likes of systemd/freedesktop centric apps. - start=$(date +%s) + declare start=$(date +%s) ### Write config summary LOG_SECTION="config_summary" do_with_logging write_config_summary_output_file # Check and install dependencies, directory structure and settings - LOG_SECTION="prepare_host" do_with_logging prepare_host + prepare_host # this has its own logging sections, and is possibly interactive. # Aggregate packages, in its own logging section; this decides internally on KERNEL_ONLY=no LOG_SECTION="aggregate_packages" do_with_logging aggregate_packages @@ -44,7 +44,8 @@ function main_default_build_single() { fi if [[ $CLEAN_LEVEL == *sources* ]]; then - general_cleaning "sources" + # early cleaning for sources, since fetch_and_build_host_tools() uses it. + LOG_SECTION="cleaning_early_sources" do_with_logging general_cleaning "sources" fi # Too many things being done. Allow doing only one thing. For core development, mostly. @@ -72,20 +73,7 @@ function main_default_build_single() { LOG_SECTION="fetch_and_build_host_tools" do_with_logging fetch_and_build_host_tools fi - # Cleaning of old, deprecated mountpoints; only done if not running under Docker. - # mountpoints under Docker manifest as volumes, and as such can't be cleaned this way. - if [[ "${ARMBIAN_RUNNING_IN_CONTAINER}" != "yes" ]]; then - prepare_armbian_mountpoints_description_dict - local mountpoint="" - for mountpoint in "${ARMBIAN_MOUNTPOINTS_DEPRECATED[@]}"; do - local mountpoint_dir="${SRC}/${mountpoint}" - display_alert "Considering cleaning deprecated mountpoint" "${mountpoint_dir}" "debug" - if [[ -d "${mountpoint_dir}" ]]; then - display_alert "Cleaning deprecated mountpoint" "${mountpoint_dir}" "info" - run_host_command_logged rm -rf "${mountpoint_dir}" - fi - done - fi + LOG_SECTION="clean_deprecated_mountpoints" do_with_logging clean_deprecated_mountpoints for cleaning_fragment in $(tr ',' ' ' <<< "${CLEAN_LEVEL}"); do if [[ $cleaning_fragment != sources ]] && [[ $cleaning_fragment != none ]] && [[ $cleaning_fragment != make* ]]; then @@ -95,7 +83,7 @@ function main_default_build_single() { fi # Prepare ccache, cthreads, etc for the build - prepare_compilation_vars + LOG_SECTION="prepare_compilation_vars" do_with_logging prepare_compilation_vars if [[ "${do_build_uboot}" == "yes" ]]; then # Don't build u-boot at all if the BOOTCONFIG is 'none'. @@ -223,13 +211,14 @@ function main_default_build_single() { end=$(date +%s) runtime=$(((end - start) / 60)) - display_alert "Runtime" "$runtime min" "info" + # display_alert in its own logging section. + LOG_SECTION="runtime_total" do_with_logging display_alert "Runtime" "$runtime min" "info" if armbian_is_running_in_container; then BUILD_CONFIG='docker' # @TODO: this is not true, depends on how we end up launching this. fi - # Make it easy to repeat build by displaying build options used. Prepare array. + # Make it easy to repeat build by displaying build options used. Prepare array. @TODO this is inconsistent. Maybe something like the relaunch vars? local -a repeat_args=("./compile.sh" "${BUILD_CONFIG}" " BRANCH=${BRANCH}") [[ -n ${RELEASE} ]] && repeat_args+=("RELEASE=${RELEASE}") [[ -n ${BUILD_MINIMAL} ]] && repeat_args+=("BUILD_MINIMAL=${BUILD_MINIMAL}")