From b2d02071bd6703cf5350e6c6cfb7ba392d246ced Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Tue, 15 Nov 2022 01:10:06 +0100 Subject: [PATCH] armbian-next: better `ccache` logs/stats/Docker-support/use in u-boot/kernel - introduce `SHOW_CCACHE=yes` for detailed ccache statistics and logging - fix: kernel build is done under "env -i", so pass CCACHE_DIR down to Make if CCACHE_DIR is set... - split from kernel.sh, show stats also for u-boot targets; show compile time - add volume definition (under `${SRC}/cache/ccache`); auto-use that in Docker builds via `CCACHE_DIR` - better logging for `do_with_ccache_statistics()` - there's some CCACHE_DIR code from before; unify @TODO --- lib/functions/compilation/ccache.sh | 47 +++++++++++++++++++++++++++++ lib/functions/compilation/kernel.sh | 24 ++++++--------- lib/functions/compilation/uboot.sh | 7 ++++- lib/functions/host/docker.sh | 3 ++ lib/functions/host/mountpoints.sh | 2 ++ lib/functions/logging/logging.sh | 8 +++++ lib/library-functions.sh | 9 ++++++ 7 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 lib/functions/compilation/ccache.sh diff --git a/lib/functions/compilation/ccache.sh b/lib/functions/compilation/ccache.sh new file mode 100644 index 000000000..fd86588a8 --- /dev/null +++ b/lib/functions/compilation/ccache.sh @@ -0,0 +1,47 @@ +function do_with_ccache_statistics() { + display_alert "Clearing ccache statistics" "ccache" "ccache" + run_host_command_logged ccache --zero-stats + + if [[ "${SHOW_CCACHE}" == "yes" ]]; then + # show value of CCACHE_DIR + display_alert "CCACHE_DIR" "${CCACHE_DIR:-"unset"}" "ccache" + + # determine what is the actual ccache_dir in use + local ccache_dir_actual + ccache_dir_actual="$(ccache --show-config | grep "cache_dir =" | cut -d "=" -f 2 | xargs echo)" + + # calculate the size of that dir, in bytes. + local ccache_dir_size_before ccache_dir_size_after ccache_dir_size_before_human + ccache_dir_size_before="$(du -sb "${ccache_dir_actual}" | cut -f 1)" + ccache_dir_size_before_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${ccache_dir_size_before}")" + + # show the human-readable size of that dir, before we start. + display_alert "ccache dir size before" "${ccache_dir_size_before_human}" "ccache" + + # Show the ccache configuration + display_alert "ccache configuration" "ccache" "ccache" + run_host_command_logged ccache --show-config + fi + + display_alert "Running ccache'd build..." "ccache" "ccache" + "$@" + + if [[ "${SHOW_CCACHE}" == "yes" ]]; then + display_alert "Display ccache statistics" "ccache" "ccache" + run_host_command_logged ccache --show-stats --verbose + + # calculate the size of that dir, in bytes, after the compilation. + ccache_dir_size_after="$(du -sb "${ccache_dir_actual}" | cut -f 1)" + + # calculate the difference, in bytes. + local ccache_dir_size_diff + ccache_dir_size_diff="$((ccache_dir_size_after - ccache_dir_size_before))" + + # calculate the difference, in human-readable format; numfmt is from coreutils. + local ccache_dir_size_diff_human + ccache_dir_size_diff_human="$(numfmt --to=iec-i --suffix=B --format="%.2f" "${ccache_dir_size_diff}")" + + # display the difference + display_alert "ccache dir size change" "${ccache_dir_size_diff_human}" "ccache" + fi +} diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index 61649782e..dcccbfb55 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -10,7 +10,6 @@ function run_kernel_make_internal() { declare -a -g DISTCC_MAKE_J_PARALLEL=() prepare_distcc_compilation_config - common_make_envs=( "CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate "PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH. @@ -19,14 +18,19 @@ function run_kernel_make_internal() { "TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work. ) + # If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i" + if [[ -n "${CCACHE_DIR}" ]]; then + common_make_envs+=("CCACHE_DIR='${CCACHE_DIR}'") + fi + # Add the distcc envs, if any. common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}") common_make_params_quoted=( # @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir. - + "${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled. - + "ARCH=${ARCHITECTURE}" # Key param. Everything depends on this. "LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change to "localversion" file @@ -35,8 +39,8 @@ function run_kernel_make_internal() { "SOURCE_DATE_EPOCH=${kernel_base_revision_ts}" # https://reproducible-builds.org/docs/source-date-epoch/ and https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html "KBUILD_BUILD_TIMESTAMP=${kernel_base_revision_date}" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-timestamp - "KBUILD_BUILD_USER=armbian-build" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host - "KBUILD_BUILD_HOST=armbian-bm" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host + "KBUILD_BUILD_USER=armbian" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host + "KBUILD_BUILD_HOST=next" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host "KGZIP=pigz" "KBZIP2=pbzip2" # Parallel compression, use explicit parallel compressors https://lore.kernel.org/lkml/20200901151002.988547791@linuxfoundation.org/ ) @@ -472,13 +476,3 @@ function kernel_build_and_package() { display_alert "Kernel built and packaged in" "$((SECONDS - ts)) seconds - ${version}-${LINUXFAMILY}" "info" } - -function do_with_ccache_statistics() { - display_alert "Clearing ccache statistics" "ccache" "debug" - ccache --zero-stats - - "$@" - - display_alert "Display ccache statistics" "ccache" "debug" - run_host_command_logged ccache --show-stats -} diff --git a/lib/functions/compilation/uboot.sh b/lib/functions/compilation/uboot.sh index 800732574..bb9399635 100644 --- a/lib/functions/compilation/uboot.sh +++ b/lib/functions/compilation/uboot.sh @@ -156,16 +156,21 @@ function compile_uboot_target() { cross_compile="CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\"" [[ -n $UBOOT_TOOLCHAIN2 ]] && cross_compile="ARMBIAN=foe" # empty parameter is not allowed + local ts=${SECONDS} + # cflags will be passed both as CFLAGS, KCFLAGS, and both as make params and as env variables. # @TODO make configurable/expandable local uboot_cflags="-fdiagnostics-color=always -Wno-error=maybe-uninitialized -Wno-error=misleading-indentation" display_alert "${uboot_prefix}Compiling u-boot" "${version} ${target_make}" "info" export if_error_detail_message="${uboot_prefix}Failed to build u-boot ${version} ${target_make}" - run_host_command_logged_long_running "CFLAGS='${uboot_cflags}'" "KCFLAGS='${uboot_cflags}'" \ + do_with_ccache_statistics run_host_command_logged_long_running \ + "CFLAGS='${uboot_cflags}'" "KCFLAGS='${uboot_cflags}'" \ CCACHE_BASEDIR="$(pwd)" PATH="${toolchain}:${toolchain2}:${PATH}" \ unbuffer make "$target_make" "$CTHREADS" "${cross_compile}" + display_alert "${uboot_prefix}built u-boot target" "${version} in $((SECONDS - ts)) seconds" "info" + if [[ $(type -t uboot_custom_postprocess) == function ]]; then display_alert "${uboot_prefix}Postprocessing u-boot" "${version} ${target_make}" uboot_custom_postprocess diff --git a/lib/functions/host/docker.sh b/lib/functions/host/docker.sh index bb6905d2b..828578dbe 100755 --- a/lib/functions/host/docker.sh +++ b/lib/functions/host/docker.sh @@ -291,6 +291,9 @@ function docker_cli_prepare_launch() { # Pass env var ARMBIAN_RUNNING_IN_CONTAINER to indicate we're running under Docker. This is also set in the Dockerfile; make sure. "--env" "ARMBIAN_RUNNING_IN_CONTAINER=yes" + # Change the ccache directory to the named volume or bind created. + "--env" "CCACHE_DIR=${DOCKER_ARMBIAN_TARGET_PATH}/cache/ccache" + # @TODO: if user on terminal, pass the TERM down to the container. "--env" "TERM=${TERM}" # @TODO: pass down the CI-related env vars. diff --git a/lib/functions/host/mountpoints.sh b/lib/functions/host/mountpoints.sh index 523090bd8..9f7d334a0 100644 --- a/lib/functions/host/mountpoints.sh +++ b/lib/functions/host/mountpoints.sh @@ -9,6 +9,7 @@ function prepare_armbian_mountpoints_description_dict() { "cache/aptcache" "cache/rootfs" "cache/initrd" "cache/sources" "cache/sources/linux-kernel" + "cache/ccache" ) declare -A -g ARMBIAN_MOUNTPOINTS_DESC_DICT=( @@ -25,6 +26,7 @@ function prepare_armbian_mountpoints_description_dict() { ["cache/initrd"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # initrd.img cache, can be bind-mounted or a volume. On Darwin it's too slow to bind-mount, so it's a volume by default. On Linux, it's a bind-mount by default. ["cache/sources"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # operating directory. many things are cloned in here, and some are even built inside. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. ["cache/sources/linux-kernel"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # working tree for kernel builds. huge. contains both sources and the built object files. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. + ["cache/ccache"]="docker_kind_linux=bind docker_kind_darwin=namedvolume" # ccache object store. limited to 5gb by default. needs to be local to the container, so it's a volume by default. On Linux, it's a bind-mount by default. ) } diff --git a/lib/functions/logging/logging.sh b/lib/functions/logging/logging.sh index 9dede6663..4215e0984 100644 --- a/lib/functions/logging/logging.sh +++ b/lib/functions/logging/logging.sh @@ -227,6 +227,14 @@ function display_alert() { inline_logs_color="${tool_color}" # either gray or normal, a bit subdued. ;; + ccache) + if [[ "${SHOW_CCACHE}" != "yes" ]]; then # ccache-related debugging messages, very very verbose + skip_screen=1 + fi + level_indicator="🙈" + inline_logs_color="\e[1;34m" # blue; 36 would be cyan + ;; + aggregation) if [[ "${SHOW_AGGREGATION}" != "yes" ]]; then # aggregation (PACKAGE LISTS), very very verbose skip_screen=1 diff --git a/lib/library-functions.sh b/lib/library-functions.sh index 4e7c55dd9..bab4871db 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -136,6 +136,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/compilation/atf.sh source "${SRC}"/lib/functions/compilation/atf.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/compilation/ccache.sh +# shellcheck source=lib/functions/compilation/ccache.sh +source "${SRC}"/lib/functions/compilation/ccache.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled