mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
armbian-next: introduce tmpfs-utils.sh; put LOGDIR and WORKDIR under tmpfs; set CCACHE_TEMPDIR under WORKDIR
- introduce generic `prepare_tmpfs_for()`, which manages it's own cleanup/dir removal - use it for `WORKDIR` (which is `TMPDIR`) and `LOGDIR` - adapt previous cleanup handlers for those, so they delete their contents but not the dir itself (which might be mounted) - also: make `ARMBIAN_LOG_CLI_ID` readonly together with other superglobals - set `XDG_RUNTIME_DIR` & `XDG_RUNTIME_DIR` together with `TMPDIR` - kernel-make.sh: pass `CCACHE_TEMPDIR` down to Kernel make (thanks @the-Going)
This commit is contained in:
Ricardo Pardini
@@ -108,9 +108,17 @@ function cli_entrypoint() {
|
||||
declare -g -r MOUNT="${WORKDIR_BASE_TMP}/mount-${ARMBIAN_BUILD_UUID}" # MOUNT ("mounted on the loop") is the mounted root on final image (via loop). "image" stage
|
||||
declare -g -r DESTIMG="${WORKDIR_BASE_TMP}/image-${ARMBIAN_BUILD_UUID}" # DESTIMG is where the backing image (raw, huge, sparse file) is kept (not the final destination)
|
||||
|
||||
LOG_SECTION="entrypoint" start_logging_section # This creates LOGDIR. @TODO: also maybe causes a spurious group to be created in the log file
|
||||
# Make sure ARMBIAN_LOG_CLI_ID is set, and unique, and readonly.
|
||||
# Pre-runs might change it before this, but if not set, default to ARMBIAN_COMMAND.
|
||||
declare -r -g ARMBIAN_LOG_CLI_ID="${ARMBIAN_LOG_CLI_ID:-${ARMBIAN_COMMAND}}"
|
||||
|
||||
# If we're on Linux & root, mount tmpfs on LOGDIR. This has it's own cleanup handler.
|
||||
# It also _creates_ the LOGDIR, and the cleanup handler will delete.
|
||||
prepare_tmpfs_for "LOGDIR" "${LOGDIR}"
|
||||
|
||||
LOG_SECTION="entrypoint" start_logging_section # This will create LOGDIR if it does not exist. @TODO: also maybe causes a spurious group to be created in the log file
|
||||
add_cleanup_handler trap_handler_cleanup_logging # cleanup handler for logs; it rolls it up from LOGDIR into DEST/logs
|
||||
add_cleanup_handler trap_handler_reset_output_owner # make sure output folder is owned by pre-sudo user if that's the case
|
||||
add_cleanup_handler trap_handler_reset_output_owner # make sure output folder is owned by pre-sudo/pre-Docker user if that's the case
|
||||
|
||||
# @TODO: So gigantic contention point here about logging the basic deps installation.
|
||||
if [[ "${ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS}" == "yes" ]]; then
|
||||
|
||||
@@ -11,11 +11,12 @@ function run_kernel_make_internal() {
|
||||
prepare_distcc_compilation_config
|
||||
|
||||
common_make_envs=(
|
||||
"CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
|
||||
"PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH.
|
||||
"DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove?
|
||||
"XZ_OPT='--threads=0'" # Use parallel XZ compression
|
||||
"TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work.
|
||||
"CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
|
||||
"CCACHE_TEMPDIR=\"${CCACHE_TEMPDIR:?}\"" # Temporary directory for ccache, under WORKDIR
|
||||
"PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH.
|
||||
"DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove?
|
||||
"XZ_OPT='--threads=0'" # Use parallel XZ compression
|
||||
"TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work.
|
||||
)
|
||||
|
||||
# If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i"
|
||||
@@ -32,7 +33,7 @@ function run_kernel_make_internal() {
|
||||
"${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled.
|
||||
|
||||
"ARCH=${ARCHITECTURE}" # Key param. Everything depends on this.
|
||||
"LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change to "localversion" file
|
||||
"LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change hack at .config; that might handles mtime better
|
||||
|
||||
"CROSS_COMPILE=${CCACHE} ${DISTCC_CROSS_COMPILE_PREFIX[@]} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make
|
||||
"KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning
|
||||
@@ -42,7 +43,7 @@ function run_kernel_make_internal() {
|
||||
"KBUILD_BUILD_USER=armbian" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
|
||||
"KBUILD_BUILD_HOST=next" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
|
||||
|
||||
"KGZIP=pigz" "KBZIP2=pbzip2" # Parallel compression, use explicit parallel compressors https://lore.kernel.org/lkml/20200901151002.988547791@linuxfoundation.org/
|
||||
"KGZIP=pigz" "KBZIP2=pbzip2" # Parallel compression, use explicit parallel compressors https://lore.kernel.org/lkml/20200901151002.988547791@linuxfoundation.org/ # @TODO: what about XZ?
|
||||
)
|
||||
|
||||
# last statement, so it passes the result to calling function. "env -i" is used for empty env
|
||||
|
||||
87
lib/functions/host/tmpfs-utils.sh
Normal file
87
lib/functions/host/tmpfs-utils.sh
Normal file
@@ -0,0 +1,87 @@
|
||||
# Call: prepare_tmpfs_for "NAME_OF_TMPFS_DIR" "${PATH_TO_DIR}" # this adds its own cleanup handler
|
||||
function prepare_tmpfs_for() {
|
||||
declare tmpfs_name="${1}"
|
||||
declare tmpfs_path="${2}"
|
||||
# validate parameters
|
||||
if [[ -z "${tmpfs_name}" ]]; then
|
||||
exit_with_error "prepare_tmpfs_for: tmpfs_name (arg 1) is empty"
|
||||
fi
|
||||
if [[ -z "${tmpfs_path}" ]]; then
|
||||
exit_with_error "prepare_tmpfs_for: tmpfs_path (arg 2) is empty"
|
||||
fi
|
||||
|
||||
# params for the handler; initially just repeat our own params
|
||||
declare -a cleanup_params=("${tmpfs_name}" "${tmpfs_path}")
|
||||
|
||||
# create the dir if not exists and note that down for the cleanup handler
|
||||
if [[ ! -d "${tmpfs_path}" ]]; then
|
||||
display_alert "prepare_tmpfs_for: creating dir" "${tmpfs_path}" "cleanup"
|
||||
mkdir -p "${tmpfs_path}"
|
||||
cleanup_params+=("remove_dir")
|
||||
else
|
||||
display_alert "prepare_tmpfs_for: dir exists" "${tmpfs_path}" "cleanup"
|
||||
cleanup_params+=("no_remove_dir")
|
||||
fi
|
||||
|
||||
# Do nothing if we're not on Linux (detect via OSTYPE) and root. Still, setup the cleanup handler.
|
||||
if [[ "${OSTYPE}" != "linux"* ]] || [[ "${EUID}" -ne 0 ]]; then
|
||||
display_alert "prepare_tmpfs_for: not on Linux or not root, skipping" "${tmpfs_name}" "cleanup"
|
||||
cleanup_params+=("no_umount_tmpfs")
|
||||
elif [[ "${ARMBIAN_INSIDE_DOCKERFILE_BUILD}" == "yes" ]]; then
|
||||
display_alert "prepare_tmpfs_for: inside Dockerfile build, skipping" "${tmpfs_name}" "cleanup"
|
||||
cleanup_params+=("no_umount_tmpfs")
|
||||
else
|
||||
display_alert "prepare_tmpfs_for: on Linux and root, MOUNTING TMPFS" "${tmpfs_name}" "cleanup"
|
||||
# mount tmpfs on it
|
||||
mount -t tmpfs tmpfs "${tmpfs_path}"
|
||||
cleanup_params+=("umount_tmpfs")
|
||||
|
||||
#cleanup_params+=("no_umount_tmpfs")
|
||||
fi
|
||||
|
||||
# add the cleanup handler
|
||||
declare cleanup_handler="cleanup_tmpfs_for ${cleanup_params[*]@Q}"
|
||||
display_alert "prepare_tmpfs_for: add cleanup handler" "${cleanup_handler}" "cleanup"
|
||||
add_cleanup_handler "${cleanup_handler}"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function cleanup_tmpfs_for() {
|
||||
declare tmpfs_name="${1}"
|
||||
declare tmpfs_path="${2}"
|
||||
declare remove_dir="${3}"
|
||||
declare umount_tmpfs="${4}"
|
||||
|
||||
# validate parameters
|
||||
if [[ -z "${tmpfs_name}" ]]; then
|
||||
exit_with_error "cleanup_tmpfs_for: tmpfs_name (arg 1) is empty"
|
||||
fi
|
||||
if [[ -z "${tmpfs_path}" ]]; then
|
||||
exit_with_error "cleanup_tmpfs_for: tmpfs_path (arg 2) is empty"
|
||||
fi
|
||||
if [[ -z "${remove_dir}" ]]; then
|
||||
exit_with_error "cleanup_tmpfs_for: remove_dir (arg 3) is empty"
|
||||
fi
|
||||
if [[ -z "${umount_tmpfs}" ]]; then
|
||||
exit_with_error "cleanup_tmpfs_for: umount_tmpfs (arg 4) is empty"
|
||||
fi
|
||||
|
||||
# umount tmpfs
|
||||
if [[ "${umount_tmpfs}" == "umount_tmpfs" ]]; then
|
||||
display_alert "cleanup_tmpfs_for: umount tmpfs" "${tmpfs_name}" "cleanup"
|
||||
umount "${tmpfs_path}"
|
||||
else
|
||||
display_alert "cleanup_tmpfs_for: not umounting tmpfs" "${tmpfs_name}" "cleanup"
|
||||
fi
|
||||
|
||||
# remove the dir if we created it
|
||||
if [[ "${remove_dir}" == "remove_dir" ]]; then
|
||||
display_alert "cleanup_tmpfs_for: removing dir" "${tmpfs_path}" "cleanup"
|
||||
rm -rf "${tmpfs_path:?}"
|
||||
else
|
||||
display_alert "cleanup_tmpfs_for: not removing dir" "${tmpfs_path}" "cleanup"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This is called both early in compile.sh, but also after processing cmdline params in the cli entrypoint.sh
|
||||
# This is called both early in compile.sh, but also after processing cmdline params in the cli entrypoint.sh
|
||||
function logging_init() {
|
||||
# defaults.
|
||||
# if stdout is a terminal, then default SHOW_LOG to yes
|
||||
@@ -101,10 +101,10 @@ function print_current_asset_log_base_file() {
|
||||
}
|
||||
|
||||
function discard_logs_tmp_dir() {
|
||||
# Linux allows us to be more careful, but really, those are log files we're talking about.
|
||||
# Do not delete the dir itself, since it might be a tmpfs mount.
|
||||
if [[ "$(uname)" == "Linux" ]]; then
|
||||
rm -rf --one-file-system "${LOGDIR}"
|
||||
rm -rf --one-file-system "${LOGDIR:?}"/* # Note this is protected by :?
|
||||
else
|
||||
rm -rf "${LOGDIR}"
|
||||
rm -rf "${LOGDIR:?}"/* # Note this is protected by :?
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -7,16 +7,19 @@ function main_default_build_single() {
|
||||
fi
|
||||
|
||||
# Check the sanity of WORKDIR_BASE_TMP regarding mount options.
|
||||
check_dir_for_mount_options "${WORKDIR_BASE_TMP}" "main temporary dir"
|
||||
LOG_SECTION="check_dir_for_mount_options" do_with_logging check_dir_for_mount_options "${WORKDIR_BASE_TMP}" "main temporary dir"
|
||||
|
||||
# Starting work. Export TMPDIR, which will be picked up by all `mktemp` invocations hopefully.
|
||||
# Runner functions in logging/runners.sh will explicitly unset TMPDIR before invoking chroot.
|
||||
# Invoking chroot directly will fail in subtle ways, so, please use the runner.sh functions.
|
||||
display_alert "Starting single build, exporting TMPDIR" "${WORKDIR}" "debug"
|
||||
mkdir -p "${WORKDIR}"
|
||||
add_cleanup_handler trap_handler_cleanup_workdir
|
||||
LOG_SECTION="prepare_tmpfs_workdir" do_with_logging prepare_tmpfs_for "WORKDIR" "${WORKDIR}" # this adds its own cleanup handler, which deletes it if it was created
|
||||
add_cleanup_handler trap_handler_cleanup_workdir # this is for when it is NOT a tmpfs, for any reason; it does not delete the dir itself.
|
||||
|
||||
export TMPDIR="${WORKDIR}"
|
||||
# '-x': export
|
||||
declare -g -x TMPDIR="${WORKDIR}" # TMPDIR is default for a lot of stuff, but...
|
||||
declare -g -x CCACHE_TEMPDIR="${WORKDIR}/ccache_tmp" # Export CCACHE_TEMPDIR, under Workdir, which is hopefully under tmpfs. Thanks @the-Going for this.
|
||||
declare -g -x XDG_RUNTIME_DIR="${WORKDIR}/xdg_tmp" # XDG_RUNTIME_DIR is used by the likes of systemd/freedesktop centric apps.
|
||||
|
||||
start=$(date +%s)
|
||||
|
||||
@@ -247,10 +250,12 @@ function trap_handler_cleanup_workdir() {
|
||||
unset TMPDIR
|
||||
if [[ -d "${WORKDIR}" ]]; then
|
||||
if [[ "${PRESERVE_WORKDIR}" != "yes" ]]; then
|
||||
display_alert "Cleaning up WORKDIR" "$(du -h -s "$WORKDIR")" "debug"
|
||||
rm -rf "${WORKDIR}"
|
||||
display_alert "Cleaning up WORKDIR" "$(du -h -s "$WORKDIR")" "cleanup"
|
||||
# Remove all files and directories in WORKDIR, but not WORKDIR itself.
|
||||
rm -rf "${WORKDIR:?}"/* # Note this is protected by :?
|
||||
else
|
||||
display_alert "Preserving WORKDIR due to PRESERVE_WORKDIR=yes" "$(du -h -s "$WORKDIR")" "warn"
|
||||
# @TODO: tmpfs might just be unmounted, though.
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -550,6 +550,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
|
||||
# shellcheck source=lib/functions/host/prepare-host.sh
|
||||
source "${SRC}"/lib/functions/host/prepare-host.sh
|
||||
|
||||
# no errors tolerated. invoked before each sourced file to make sure.
|
||||
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
|
||||
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
|
||||
set -o errtrace # trace ERR through - enabled
|
||||
set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled
|
||||
### lib/functions/host/tmpfs-utils.sh
|
||||
# shellcheck source=lib/functions/host/tmpfs-utils.sh
|
||||
source "${SRC}"/lib/functions/host/tmpfs-utils.sh
|
||||
|
||||
# no errors tolerated. invoked before each sourced file to make sure.
|
||||
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
|
||||
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
|
||||
|
||||
Reference in New Issue
Block a user