mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
armbian-next: kernel: deterministic .config mtime handling; kernel build logging sections reorg
- .config is now compared with previous one byte-wise, diff shown, and mtime preserved directly via `cp -p` - this does away with the previous mtime-based BS I had written during the fasthash era (and thus remove file-mtime.sh, regen lib) - split some functions, to better control the interactive piece in the _middle_ of kernel config process - no more `kernel_config_maybe_interactive()`, now `kernel_config()` that controls its own logging sections - extract `kernel_determine_toolchain()` out of config into make - introduce internal hook `armbian_kernel_config()` that runs before custom version (for .config defaults, coming soon)
This commit is contained in:
Ricardo Pardini
@@ -1,98 +1,85 @@
|
||||
function kernel_config_maybe_interactive() {
|
||||
# Check if we're gonna do some interactive configuration; if so, don't run kernel_config under logging manager.
|
||||
if [[ $KERNEL_CONFIGURE != yes ]]; then
|
||||
LOG_SECTION="kernel_config" do_with_logging do_with_hooks kernel_config
|
||||
else
|
||||
LOG_SECTION="kernel_config_interactive" do_with_hooks kernel_config
|
||||
function kernel_config() {
|
||||
# check $kernel_work_dir is set and exists, or bail
|
||||
[[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set"
|
||||
[[ ! -d "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir does not exist: ${kernel_work_dir}"
|
||||
declare previous_config_filename=".config.armbian.previous"
|
||||
declare kernel_config_source_filename="" # which actual .config was used?
|
||||
|
||||
LOG_SECTION="kernel_config_initialize" do_with_logging do_with_hooks kernel_config_initialize
|
||||
|
||||
if [[ "${KERNEL_CONFIGURE}" == "yes" ]]; then
|
||||
# This piece is interactive, no logging
|
||||
display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug"
|
||||
run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}"
|
||||
|
||||
# Export, but log about it too.
|
||||
LOG_SECTION="kernel_config_export" do_with_logging do_with_hooks kernel_config_export
|
||||
fi
|
||||
|
||||
LOG_SECTION="kernel_config_finalize" do_with_logging do_with_hooks kernel_config_finalize
|
||||
}
|
||||
|
||||
function kernel_config() {
|
||||
# re-read kernel version after patching
|
||||
version=$(grab_version "$kernel_work_dir")
|
||||
function kernel_config_initialize() {
|
||||
display_alert "Configuring kernel" "${LINUXCONFIG}" "info"
|
||||
|
||||
display_alert "Compiling $BRANCH kernel" "$version" "info"
|
||||
|
||||
# compare with the architecture of the current Debian node
|
||||
# if it matches we use the system compiler
|
||||
if dpkg-architecture -e "${ARCH}"; then
|
||||
display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
||||
else
|
||||
display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
||||
toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC")
|
||||
[[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC"
|
||||
# If a `.config` already exists (from previous build), store it, preserving date.
|
||||
# We will compare the result of the new configuration to it, and if the contents are the same, we'll restore the original date.
|
||||
# This way we avoid unnecessary recompilation of the kernel; even if the .config contents
|
||||
# have not changed, the date will be different, and Kbuild will at least re-link everything.
|
||||
if [[ -f "${kernel_work_dir}/.config" ]]; then
|
||||
display_alert "Preserving previous kernel configuration" "${previous_config_filename}" "debug"
|
||||
run_host_command_logged cp -pv "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}"
|
||||
fi
|
||||
|
||||
kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)"
|
||||
display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info"
|
||||
|
||||
# copy kernel config
|
||||
local COPY_CONFIG_BACK_TO=""
|
||||
|
||||
if [[ $KERNEL_KEEP_CONFIG == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then
|
||||
display_alert "Using previous kernel config" "${DEST}/config/$LINUXCONFIG.config" "info"
|
||||
# copy kernel config from configuration, userpatches
|
||||
if [[ "${KERNEL_KEEP_CONFIG}" == yes && -f "${DEST}"/config/$LINUXCONFIG.config ]]; then
|
||||
display_alert "Using previously-exported kernel config" "${DEST}/config/$LINUXCONFIG.config" "info"
|
||||
run_host_command_logged cp -pv "${DEST}/config/${LINUXCONFIG}.config" .config
|
||||
else
|
||||
# @TODO: rpardini: this is too contrived, make obvious, use an array and a loop and stop repeating itself
|
||||
# @TODO: rpardini: this is too contrived
|
||||
if [[ -f $USERPATCHES_PATH/$LINUXCONFIG.config ]]; then
|
||||
display_alert "Using kernel config provided by user" "userpatches/$LINUXCONFIG.config" "info"
|
||||
run_host_command_logged cp -pv "${USERPATCHES_PATH}/${LINUXCONFIG}.config" .config
|
||||
COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/${LINUXCONFIG}.config"
|
||||
kernel_config_source_filename="${USERPATCHES_PATH}/${LINUXCONFIG}.config"
|
||||
elif [[ -f "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" ]]; then
|
||||
display_alert "Using kernel config provided by user in config/kernel folder" "config/kernel/${LINUXCONFIG}.config" "info"
|
||||
run_host_command_logged cp -pv "${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config" .config
|
||||
COPY_CONFIG_BACK_TO="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config"
|
||||
kernel_config_source_filename="${USERPATCHES_PATH}/config/kernel/${LINUXCONFIG}.config"
|
||||
else
|
||||
display_alert "Using kernel config file" "config/kernel/$LINUXCONFIG.config" "info"
|
||||
run_host_command_logged cp -pv "${SRC}/config/kernel/${LINUXCONFIG}.config" .config
|
||||
COPY_CONFIG_BACK_TO="${SRC}/config/kernel/${LINUXCONFIG}.config"
|
||||
kernel_config_source_filename="${SRC}/config/kernel/${LINUXCONFIG}.config"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Store the .config modification date at this time, for restoring later. Otherwise rebuilds.
|
||||
local kernel_config_mtime
|
||||
kernel_config_mtime=$(get_file_modification_time ".config")
|
||||
# Start by running olddefconfig -- always.
|
||||
# It "updates" the config, using defaults from Kbuild files in the source tree.
|
||||
# It is worthy noting that on the first run, it builds the tools, so the host-side compiler has to be working,
|
||||
# regardless of the cross-build toolchain.
|
||||
run_kernel_make olddefconfig
|
||||
|
||||
# Run the core-armbian config modifications here, built-in extensions:
|
||||
# 1) Enable the options for the extrawifi/drivers; so we don't need to worry about them when updating configs
|
||||
# 2) Disable: debug, version shenanigans, signing, and compression of modules, to ensure sanity
|
||||
call_extension_method "armbian_kernel_config" <<- 'ARMBIAN_KERNEL_CONFIG'
|
||||
*Armbian-core default hook point for pre-olddefconfig Kernel config modifications*
|
||||
NOT for user consumption. Do NOT use this hook, this is internal to Armbian.
|
||||
Instead, use `custom_kernel_config` which runs later and can undo anything done by this step.
|
||||
ARMBIAN_KERNEL_CONFIG
|
||||
|
||||
# Custom hooks receive a clean / updated config; depending on their modifications, they may need to run olddefconfig again.
|
||||
call_extension_method "custom_kernel_config" <<- 'CUSTOM_KERNEL_CONFIG'
|
||||
*Kernel .config is in place, still clean from git version*
|
||||
Called after ${LINUXCONFIG}.config is put in place (.config).
|
||||
Before any olddefconfig any Kconfig make is called.
|
||||
A good place to customize the .config directly.
|
||||
Armbian default Kconfig modifications have already been applied and can be overriden.
|
||||
CUSTOM_KERNEL_CONFIG
|
||||
|
||||
display_alert "Kernel configuration" "${LINUXCONFIG}" "info"
|
||||
|
||||
if [[ $KERNEL_CONFIGURE != yes ]]; then
|
||||
run_kernel_make olddefconfig # @TODO: what is this? does it fuck up dates?
|
||||
else
|
||||
display_alert "Starting (non-interactive) kernel olddefconfig" "${LINUXCONFIG}" "debug"
|
||||
|
||||
run_kernel_make olddefconfig
|
||||
|
||||
# No logging for this. this is UI piece
|
||||
display_alert "Starting (interactive) kernel ${KERNEL_MENUCONFIG:-menuconfig}" "${LINUXCONFIG}" "debug"
|
||||
run_kernel_make_dialog "${KERNEL_MENUCONFIG:-menuconfig}"
|
||||
|
||||
# Capture new date. Otherwise changes not detected by make.
|
||||
kernel_config_mtime=$(get_file_modification_time ".config")
|
||||
|
||||
# store kernel config in easily reachable place
|
||||
mkdir -p "${DEST}"/config
|
||||
display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info"
|
||||
run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config"
|
||||
|
||||
# store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done.
|
||||
if [[ "${COPY_CONFIG_BACK_TO}" != "" ]]; then
|
||||
display_alert "Exporting new kernel config - git commit pending" "${COPY_CONFIG_BACK_TO}" "info"
|
||||
run_host_command_logged cp -pv .config "${COPY_CONFIG_BACK_TO}"
|
||||
|
||||
# export defconfig
|
||||
run_kernel_make savedefconfig
|
||||
run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig"
|
||||
run_host_command_logged cp -pv defconfig "${COPY_CONFIG_BACK_TO}.defconfig"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function kernel_config_finalize() {
|
||||
call_extension_method "custom_kernel_config_post_defconfig" <<- 'CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG'
|
||||
*Kernel .config is in place, already processed by Armbian*
|
||||
Called after ${LINUXCONFIG}.config is put in place (.config).
|
||||
@@ -100,6 +87,38 @@ function kernel_config() {
|
||||
A good place to customize the .config last-minute.
|
||||
CUSTOM_KERNEL_CONFIG_POST_DEFCONFIG
|
||||
|
||||
# Restore the date of .config. Above delta is a pure function, theoretically.
|
||||
set_files_modification_time "${kernel_config_mtime}" ".config"
|
||||
# Now, compare the .config with the previous one, and if they are the same, restore the original date.
|
||||
# This way we avoid unnecessary recompilation of the kernel; even if the .config contents
|
||||
# have not changed, the date will be different, and Kbuild will at least re-link everything.
|
||||
if [[ -f "${kernel_work_dir}/${previous_config_filename}" ]]; then
|
||||
# from "man cmp": Exit status is 0 if inputs are the same, 1 if different, 2 if trouble.
|
||||
if cmp "${kernel_work_dir}/.config" "${kernel_work_dir}/${previous_config_filename}"; then
|
||||
display_alert "Kernel configuration unchanged from previous run" "optimizing for fast rebuilds" "cachehit"
|
||||
run_host_command_logged cp -pv "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config"
|
||||
else
|
||||
display_alert "Kernel configuration changed from previous build" "optimizing for correctness" "info"
|
||||
# ad: added lines; de: deleted lines; hd: header lines; ln: line numbers
|
||||
# run_host_command_logged diff -u --color=always "--palette='rs=0:hd=1:ad=33:de=37:ln=36'" "${kernel_work_dir}/${previous_config_filename}" "${kernel_work_dir}/.config" "|| true" # no errors please
|
||||
fi
|
||||
# either way, remove the previous file
|
||||
run_host_command_logged rm -f "${kernel_work_dir}/${previous_config_filename}"
|
||||
fi
|
||||
}
|
||||
|
||||
function kernel_config_export() {
|
||||
# store kernel config in easily reachable place
|
||||
mkdir -p "${DEST}"/config
|
||||
display_alert "Exporting new kernel config" "$DEST/config/$LINUXCONFIG.config" "info"
|
||||
run_host_command_logged cp -pv .config "${DEST}/config/${LINUXCONFIG}.config"
|
||||
|
||||
# store back into original LINUXCONFIG too, if it came from there, so it's pending commits when done.
|
||||
if [[ "${kernel_config_source_filename}" != "" ]]; then
|
||||
display_alert "Exporting new kernel config - git commit pending" "${kernel_config_source_filename}" "info"
|
||||
run_host_command_logged cp -pv .config "${kernel_config_source_filename}"
|
||||
|
||||
# export defconfig
|
||||
run_kernel_make savedefconfig
|
||||
run_host_command_logged cp -pv defconfig "${DEST}/config/${LINUXCONFIG}.defconfig"
|
||||
run_host_command_logged cp -pv defconfig "${kernel_config_source_filename}.defconfig"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -65,3 +65,17 @@ function run_kernel_make_long_running() {
|
||||
display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug"
|
||||
}
|
||||
|
||||
function kernel_determine_toolchain() {
|
||||
# compare with the architecture of the current Debian node
|
||||
# if it matches we use the system compiler
|
||||
if dpkg-architecture -e "${ARCH}"; then
|
||||
display_alert "Native compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
||||
else
|
||||
display_alert "Cross compilation" "target ${ARCH} on host $(dpkg --print-architecture)"
|
||||
toolchain=$(find_toolchain "$KERNEL_COMPILER" "$KERNEL_USE_GCC")
|
||||
[[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${KERNEL_COMPILER}gcc $KERNEL_USE_GCC"
|
||||
fi
|
||||
|
||||
kernel_compiler_version="$(eval env PATH="${toolchain}:${PATH}" "${KERNEL_COMPILER}gcc" -dumpfullversion -dumpversion)"
|
||||
display_alert "Compiler version" "${KERNEL_COMPILER}gcc ${kernel_compiler_version}" "info"
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@ function compile_kernel() {
|
||||
LOG_SECTION="kernel_maybe_clean" do_with_logging do_with_hooks kernel_maybe_clean
|
||||
|
||||
# Patching.
|
||||
local version hash pre_patch_version
|
||||
kernel_main_patching
|
||||
declare hash pre_patch_version
|
||||
kernel_main_patching # has it's own logging sections inside
|
||||
|
||||
# Stop after patching;
|
||||
if [[ "${PATCH_ONLY}" == yes ]]; then
|
||||
@@ -35,23 +35,30 @@ function compile_kernel() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
local toolchain
|
||||
kernel_config_maybe_interactive
|
||||
# re-read kernel version after patching
|
||||
declare version
|
||||
version=$(grab_version "$kernel_work_dir")
|
||||
display_alert "Compiling $BRANCH kernel" "$version" "info"
|
||||
|
||||
# determine the toolchain
|
||||
declare toolchain
|
||||
LOG_SECTION="kernel_determine_toolchain" do_with_logging do_with_hooks kernel_determine_toolchain
|
||||
|
||||
kernel_config # has it's own logging sections inside
|
||||
|
||||
# package the kernel-source .deb
|
||||
LOG_SECTION="kernel_package_source" do_with_logging do_with_hooks kernel_package_source
|
||||
|
||||
# build via make and package .debs; they're separate sub-steps
|
||||
LOG_SECTION="kernel_build_and_package" do_with_logging do_with_hooks kernel_build_and_package
|
||||
kernel_prepare_build_and_package # has it's own logging sections inside
|
||||
|
||||
display_alert "Done with" "kernel compile" "debug"
|
||||
cd "${kernel_work_dir}/.." || exit
|
||||
|
||||
rm -f linux-firmware-image-*.deb # remove firmware image packages here - easier than patching ~40 packaging scripts at once
|
||||
run_host_command_logged rsync --remove-source-files -r ./*.deb "${DEB_STORAGE}/"
|
||||
LOG_SECTION="kernel_deploy_pkg" do_with_logging do_with_hooks kernel_deploy_pkg
|
||||
|
||||
# kernel build worked; let's clean up the git-bundle cache, since the git-bare cache is proven working.
|
||||
kernel_cleanup_bundle_artifacts
|
||||
# @TODO: armbian-oleg: clean this earlier, so we save some 2gb on disk _during_ kernel compile, not after
|
||||
LOG_SECTION="kernel_cleanup_bundle_artifacts" do_with_logging do_with_hooks kernel_cleanup_bundle_artifacts
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -115,14 +122,14 @@ function kernel_package_source() {
|
||||
display_alert "$(basename "${sources_pkg_dir}.deb" ".deb") packaged" "$((SECONDS - ts)) seconds, ${tarball_size} tarball, ${package_size} .deb" "info"
|
||||
}
|
||||
|
||||
function kernel_build_and_package() {
|
||||
local ts=${SECONDS}
|
||||
|
||||
cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
|
||||
|
||||
local -a build_targets=("all") # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH}
|
||||
function kernel_prepare_build_and_package() {
|
||||
declare -a build_targets
|
||||
declare kernel_dest_install_dir
|
||||
kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.temp.install.target.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up.
|
||||
declare -a install_make_params_quoted
|
||||
declare -A kernel_install_dirs
|
||||
|
||||
build_targets=("all") # "All" builds the vmlinux/Image/Image.gz default for the ${ARCH}
|
||||
kernel_dest_install_dir=$(mktemp -d "${WORKDIR}/kernel.XXXXXXXXX") # subject to TMPDIR/WORKDIR, so is protected by single/common error trapmanager to clean-up.
|
||||
|
||||
# define dict with vars passed and target directories
|
||||
declare -A kernel_install_dirs=(
|
||||
@@ -139,7 +146,6 @@ function kernel_build_and_package() {
|
||||
fi
|
||||
|
||||
# loop over the keys above, get the value, create param value in array; also mkdir the dir
|
||||
declare -a install_make_params_quoted
|
||||
local dir_key
|
||||
for dir_key in "${!kernel_install_dirs[@]}"; do
|
||||
local dir="${kernel_install_dirs["${dir_key}"]}"
|
||||
@@ -148,14 +154,38 @@ function kernel_build_and_package() {
|
||||
install_make_params_quoted+=("${value}")
|
||||
done
|
||||
|
||||
# Fire off the build & package
|
||||
LOG_SECTION="kernel_build" do_with_logging do_with_hooks kernel_build
|
||||
|
||||
LOG_SECTION="kernel_package" do_with_logging do_with_hooks kernel_package
|
||||
}
|
||||
|
||||
function kernel_build() {
|
||||
local ts=${SECONDS}
|
||||
cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
|
||||
|
||||
display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info"
|
||||
make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \
|
||||
do_with_ccache_statistics \
|
||||
run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}"
|
||||
|
||||
display_alert "Kernel built in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
|
||||
}
|
||||
|
||||
function kernel_package() {
|
||||
local ts=${SECONDS}
|
||||
cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
|
||||
display_alert "Packaging kernel" "${LINUXFAMILY} ${LINUXCONFIG}" "info"
|
||||
prepare_kernel_packaging_debs "${kernel_work_dir}" "${kernel_dest_install_dir}" "${version}" kernel_install_dirs
|
||||
|
||||
display_alert "Kernel built and packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
|
||||
display_alert "Kernel packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info"
|
||||
}
|
||||
|
||||
function kernel_deploy_pkg() {
|
||||
cd "${kernel_work_dir}/.." || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}"
|
||||
|
||||
# @TODO: rpardini: this is kept for historical reasons... wth?
|
||||
# remove firmware image packages here - easier than patching ~40 packaging scripts at once
|
||||
run_host_command_logged rm -fv "linux-firmware-image-*.deb"
|
||||
|
||||
run_host_command_logged rsync -v --remove-source-files -r ./*.deb "${DEB_STORAGE}/"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user