From 9370c19a4470e72113dc6dbe5d363500a20d0c9a Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Tue, 27 Dec 2022 21:27:16 +0100 Subject: [PATCH] armbian-next: drop old code for patching & fasthash - split the modification time stuff to general --- lib/functions/compilation/kernel-git.sh | 1 - lib/functions/compilation/kernel-patching.sh | 94 ------------------- lib/functions/compilation/kernel.sh | 3 - lib/functions/compilation/patch/patching.sh | 63 +++---------- lib/functions/compilation/uboot.sh | 3 - .../fasthash.sh => general/file-mtime.sh} | 44 --------- lib/functions/general/git.sh | 1 - 7 files changed, 11 insertions(+), 198 deletions(-) rename lib/functions/{compilation/patch/fasthash.sh => general/file-mtime.sh} (58%) diff --git a/lib/functions/compilation/kernel-git.sh b/lib/functions/compilation/kernel-git.sh index d819485cc..2ea8d5e94 100644 --- a/lib/functions/compilation/kernel-git.sh +++ b/lib/functions/compilation/kernel-git.sh @@ -170,7 +170,6 @@ function kernel_prepare_git_pre_fetch() { function kernel_prepare_git() { [[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then? - [[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree" display_alert "Downloading sources" "kernel" "git" diff --git a/lib/functions/compilation/kernel-patching.sh b/lib/functions/compilation/kernel-patching.sh index f31124174..12b13fe04 100644 --- a/lib/functions/compilation/kernel-patching.sh +++ b/lib/functions/compilation/kernel-patching.sh @@ -54,10 +54,6 @@ function kernel_main_patching() { # Python patching will git reset to the kernel SHA1 git revision, and remove all untracked files. LOG_SECTION="kernel_main_patching_python" do_with_logging do_with_hooks kernel_main_patching_python - # The old way... - #LOG_SECTION="kernel_prepare_patching" do_with_logging do_with_hooks kernel_prepare_patching - #LOG_SECTION="kernel_patching" do_with_logging do_with_hooks kernel_patching - # STOP HERE, for cli support for patching tools. if [[ "${PATCH_ONLY}" == "yes" ]]; then return 0 @@ -69,93 +65,3 @@ function kernel_main_patching() { return 0 # there is a shortcircuit above } -function kernel_prepare_patching() { - if [[ $USE_OVERLAYFS == yes ]]; then # @TODO: when is this set to yes? - display_alert "Using overlayfs_wrapper" "kernel_${LINUXFAMILY}_${BRANCH}" "debug" - kernel_work_dir=$(overlayfs_wrapper "wrap" "$SRC/cache/sources/$LINUXSOURCEDIR" "kernel_${LINUXFAMILY}_${BRANCH}") - fi - cd "${kernel_work_dir}" || exit - - # @TODO: why would we delete localversion? - # @TODO: it should be the opposite, writing localversion to disk, _instead_ of passing it via make. - # @TODO: if it turns out to be the case, do a commit with it... (possibly later, after patching?) - rm -f localversion - - # read kernel version - version=$(grab_version "$kernel_work_dir") - pre_patch_version="${version}" - display_alert "Pre-patch kernel version" "${pre_patch_version}" "debug" - - # read kernel git hash - hash=$(git --git-dir="$kernel_work_dir"/.git rev-parse HEAD) -} - -function kernel_patching() { - ## Start kernel patching process. - ## There's a few objectives here: - ## - (always) produce a fasthash: represents "what would be done" (eg: md5 of a patch, crc32 of description). - ## - (optionally) execute modification against living tree (eg: apply a patch, copy a file, etc). only if `DO_MODIFY=yes` - ## - (always) call mark_change_commit with the description of what was done and fasthash. - # shellcheck disable=SC2154 # declared in outer scope kernel_base_revision_mtime - declare -i patch_minimum_target_mtime="${kernel_base_revision_mtime}" - declare -i series_conf_mtime="${patch_minimum_target_mtime}" - declare -i patch_dir_mtime="${patch_minimum_target_mtime}" - display_alert "patch_minimum_target_mtime:" "${patch_minimum_target_mtime}" "debug" - - local patch_dir="${SRC}/patch/kernel/${KERNELPATCHDIR}" - local series_conf="${patch_dir}/series.conf" - - # So the minimum date has to account for removed patches; if a patch was removed from disk, the only way to reflect that - # is by looking at the parent directory's mtime, which will have been bumped. - # So we take a look at the possible directories involved here (series.conf file, and ${KERNELPATCHDIR} dir itself) - # and bump up the minimum date if that is the case. - if [[ -f "${series_conf}" ]]; then - series_conf_mtime=$(get_file_modification_time "${series_conf}") - display_alert "series.conf mtime:" "${series_conf_mtime}" "debug" - patch_minimum_target_mtime=$((series_conf_mtime > patch_minimum_target_mtime ? series_conf_mtime : patch_minimum_target_mtime)) - display_alert "patch_minimum_target_mtime after series.conf mtime:" "${patch_minimum_target_mtime}" "debug" - fi - - if [[ -d "${patch_dir}" ]]; then - patch_dir_mtime=$(get_dir_modification_time "${patch_dir}") - display_alert "patch_dir mtime:" "${patch_dir_mtime}" "debug" - patch_minimum_target_mtime=$((patch_dir_mtime > patch_minimum_target_mtime ? patch_dir_mtime : patch_minimum_target_mtime)) - display_alert "patch_minimum_target_mtime after patch_dir mtime:" "${patch_minimum_target_mtime}" "debug" - fi - - # this prepares data, and possibly creates a git branch to receive the patches. - initialize_fasthash "kernel" "${hash}" "${pre_patch_version}" "${kernel_work_dir}" - fasthash_debug "init" - - # Apply a series of patches if a series file exists - if [[ -f "${series_conf}" ]]; then - display_alert "series.conf exists. Apply" - fasthash_branch "patches-${KERNELPATCHDIR}-series.conf" - apply_patch_series "${kernel_work_dir}" "${series_conf}" # applies a series of patches, read from a file. calls process_patch_file - fi - - # applies a humongous amount of patches coming from github repos. - # it's mostly conditional, and very complex. - # @TODO: re-enable after finishing converting it with fasthash magic - # apply_kernel_patches_for_drivers "${kernel_work_dir}" "${version}" # calls process_patch_file and other stuff. there is A LOT of it. - - # @TODO: this is were "patch generation" happens? - - # Extension hook: patch_kernel_for_driver - call_extension_method "patch_kernel_for_driver" <<- 'PATCH_KERNEL_FOR_DRIVER' - *allow to add drivers/patch kernel for drivers before applying the family patches* - Patch *series* (not normal family patches) are already applied. - Useful for migrating EXTRAWIFI-related stuff to individual extensions. - Receives `${version}` and `${kernel_work_dir}` as environment variables. - PATCH_KERNEL_FOR_DRIVER - - # applies a series of patches, in directory order, from multiple directories (default/"user" patches) - # @TODO: I believe using the $BOARD here is the most confusing thing in the whole of Armbian. It should be disabled. - # @TODO: Armbian built kernels dont't vary per-board, but only per "$ARCH-$LINUXFAMILY-$BRANCH" - # @TODO: allowing for board-specific kernel patches creates insanity. uboot is enough. - fasthash_branch "patches-${KERNELPATCHDIR}-$BRANCH" - advanced_patch "kernel" "$KERNELPATCHDIR" "$BOARD" "" "$BRANCH" "$LINUXFAMILY-$BRANCH" # calls process_patch_file, "target" is empty there - - fasthash_debug "finish" - finish_fasthash "kernel" # this reports the final hash and creates git branch to build ID. All modifications commited. -} diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index 342929e63..d0a54518f 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -63,7 +63,6 @@ function kernel_maybe_clean() { cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" run_host_command_logged make ARCH="${ARCHITECTURE}" clean ) - fasthash_debug "post make clean" else display_alert "Not cleaning Kernel tree; use CLEAN_LEVEL=make-kernel if needed" "CLEAN_LEVEL=${CLEAN_LEVEL}" "debug" fi @@ -150,11 +149,9 @@ function kernel_build_and_package() { done display_alert "Building kernel" "${LINUXFAMILY} ${LINUXCONFIG} ${build_targets[*]}" "info" - fasthash_debug "build" make_filter="| grep --line-buffered -v -e 'LD' -e 'AR' -e 'INSTALL' -e 'SIGN' -e 'XZ' " \ do_with_ccache_statistics \ run_kernel_make_long_running "${install_make_params_quoted[@]@Q}" "${build_targets[@]}" - fasthash_debug "build" cd "${kernel_work_dir}" || exit_with_error "Can't cd to kernel_work_dir: ${kernel_work_dir}" display_alert "Packaging kernel" "${LINUXFAMILY} ${LINUXCONFIG}" "info" diff --git a/lib/functions/compilation/patch/patching.sh b/lib/functions/compilation/patch/patching.sh index 66f647f1b..e872c8cab 100644 --- a/lib/functions/compilation/patch/patching.sh +++ b/lib/functions/compilation/patch/patching.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # advanced_patch <{patch_dir}> # # parameters: @@ -84,64 +85,22 @@ process_patch_file() { local -i patch_date local relative_patch="${patch##"${SRC}"/}" # ${FOO##prefix} remove prefix from FOO - # report_fashtash_should_execute is report_fasthash returns true only if we're supposed to apply the patch on disk. - if report_fashtash_should_execute file "${patch}" "Apply patch ${relative_patch}"; then + # detect and remove files which patch will create + lsdiff -s --strip=1 "${patch}" | grep '^+' | awk '{print $2}' | xargs -I % sh -c 'rm -f %' - # get the modification date of the patch. make it not less than MIN_PATCH_AGE, if set. - patch_date=$(get_file_modification_time "${patch}") - # shellcheck disable=SC2154 # patch_minimum_target_mtime can be declared in outer scope - if [[ "${patch_minimum_target_mtime}" != "" ]]; then - if [[ ${patch_date} -lt ${patch_minimum_target_mtime} ]]; then - display_alert "Patch before minimum date" "${patch_date} -lt ${patch_minimum_target_mtime}" "timestamp" - patch_date=${patch_minimum_target_mtime} - fi - fi - - # detect and remove files which patch will create - lsdiff -s --strip=1 "${patch}" | grep '^+' | awk '{print $2}' | xargs -I % sh -c 'rm -f %' - - # store an array of the files that patch will add or modify, we'll set their modification times after the fact - declare -a patched_files - mapfile -t patched_files < <(lsdiff -s --strip=1 "${patch}" | grep -e '^+' -e '^!' | awk '{print $2}') - - # @TODO: try patching with `git am` first, so git contains the patch commit info/msg. -- For future git-based hashing. - # shellcheck disable=SC2015 # noted, thanks. I need to handle exit code here. - patch --batch -p1 -N --input="${patch}" --quiet --reject-file=- && { # "-" discards rejects - # Fix the dates on the patched files - set_files_modification_time "${patch_date}" "${patched_files[@]}" - display_alert "* $status ${relative_patch}" "" "info" - } || { - display_alert "* $status ${relative_patch}" "failed" "wrn" - [[ $EXIT_PATCHING_ERROR == yes ]] && exit_with_error "Aborting due to" "EXIT_PATCHING_ERROR" - } - mark_fasthash_done # will do git commit, associate fasthash to real hash. - fi + # shellcheck disable=SC2015 # noted, thanks. I need to handle exit code here. + patch --batch -p1 -N --input="${patch}" --quiet --reject-file=- && { # "-" discards rejects + display_alert "* $status ${relative_patch}" "" "info" + } || { + display_alert "* $status ${relative_patch}" "failed" "wrn" + [[ $EXIT_PATCHING_ERROR == yes ]] && exit_with_error "Aborting due to" "EXIT_PATCHING_ERROR" + } return 0 # short-circuit above, avoid exiting with error } -# apply_patch_series -apply_patch_series() { - local target_dir="${1}" - local series_file_full_path="${2}" - local included_list skip_list skip_count counter=1 base_dir - base_dir="$(dirname "${series_file_full_path}")" - included_list="$(awk '$0 !~ /^#.*|^-.*|^$/' "${series_file_full_path}")" - included_count=$(echo -n "${included_list}" | wc -w) - skip_list="$(awk '$0 ~ /^-.*/{print $NF}' "${series_file_full_path}")" - skip_count=$(echo -n "${skip_list}" | wc -w) - display_alert "apply a series of " "[$(echo -n "$included_list" | wc -w)] patches" "info" - [[ ${skip_count} -gt 0 ]] && display_alert "skipping" "[${skip_count}] patches" "warn" - cd "${target_dir}" || exit 1 - - for p in $included_list; do - process_patch_file "${base_dir}/${p}" "${counter}/${included_count}" - counter=$((counter + 1)) - done - display_alert "done applying patch series " "[$(echo -n "$included_list" | wc -w)] patches" "info" -} - userpatch_create() { + display_alert "@TODO" "@TODO armbian-next" "warn" # create commit to start from clean source git add . git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy" diff --git a/lib/functions/compilation/uboot.sh b/lib/functions/compilation/uboot.sh index 04ee1c958..8ffde4fa4 100644 --- a/lib/functions/compilation/uboot.sh +++ b/lib/functions/compilation/uboot.sh @@ -134,9 +134,6 @@ function compile_uboot_target() { fi - fasthash_debug "finish" - finish_fasthash "u-boot" # this reports the final hash and creates git branch to build ID. All modifications commited. - # workaround when two compilers are needed cross_compile="CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\"" [[ -n $UBOOT_TOOLCHAIN2 ]] && cross_compile="ARMBIAN=foe" # empty parameter is not allowed diff --git a/lib/functions/compilation/patch/fasthash.sh b/lib/functions/general/file-mtime.sh similarity index 58% rename from lib/functions/compilation/patch/fasthash.sh rename to lib/functions/general/file-mtime.sh index ded279311..6fa46065d 100644 --- a/lib/functions/compilation/patch/fasthash.sh +++ b/lib/functions/general/file-mtime.sh @@ -1,47 +1,3 @@ -function report_fashtash_should_execute() { - report_fasthash "$@" - # @TODO: if fasthash only, return 1 - return 0 -} - -function mark_fasthash_done() { - display_alert "mark_fasthash_done" "$*" "fasthash" - return 0 -} - -function report_fasthash() { - local type="${1}" - local obj="${2}" - local desc="${3}" - display_alert "report_fasthash" "${type}: ${desc}" "fasthash" - return 0 -} - -function initialize_fasthash() { - display_alert "initialize_fasthash" "$*" "fasthash" - return 0 - declare -a fast_hash_list=() # @TODO: declaring here won't do it any good, this is a shared var -} - -function fasthash_branch() { - display_alert "fasthash_branch" "$*" "fasthash" - return 0 -} - -function finish_fasthash() { - display_alert "finish_fasthash" "$*" "fasthash" - return 0 -} - -function fasthash_debug() { - if [[ "${SHOW_FASTHASH}" != "yes" ]]; then - return 0 - fi - display_alert "fasthash_debug" "$*" "fasthash" - run_host_command_logged find . -type f -printf "'%T@ %p\\n'" "|" \ - grep -v -e "\.ko" -e "\.o" -e "\.cmd" -e "\.mod" -e "\.a" -e "\.tmp" -e "\.dtb" -e ".scr" -e "\.\/debian" "|" \ - sort -n "|" tail -n 10 -} function get_file_modification_time() { # @TODO: This is almost always called from a subshell. No use throwing errors? local -i file_date diff --git a/lib/functions/general/git.sh b/lib/functions/general/git.sh index d3b171099..889425035 100644 --- a/lib/functions/general/git.sh +++ b/lib/functions/general/git.sh @@ -230,5 +230,4 @@ fetch_from_repo() { fi display_alert "Final working copy size" "$(du -h -s | awk '{print $1}')" "git" - #fasthash_debug "at the end of fetch_from_repo $dir $ref_name" }