kernel/uboot/atf: introduce kernel-patch, uboot-patch, atf-patch, uboot-config, kernel-config CLI commands; enhanced manual patching; block deprecated ways

- all interactive commands now **don't build the artifact** anymore; just patches/.configs are produced and then build ends
  - user is required to put the produced patches in the right place and build again, for full consistency
- split ATF and U-BOOT manual patching process; use CLI command `atf-patch` to patch ATF, and `uboot-patch` to patch u-boot
- non-interactive artifact builds are now 100% sans-stdin
- introduce `uboot-config` CLI command; still experimental, only produces a defconfig and not a patch
- reworked `userpatch_create()` to be (hopefully) more useful:
  - detects a previous patch and offers to apply it before continuing
  - enters a loop showing the diff, and only proceeds when user indicates he's happy with the patch
  - produces `mbox`-formatted patches via `format-patch` and standard Armbian parameters
  - uses MAINTAINER and MAINTAINERMAIL instead of git configuration (so it works in containers)
- don't allow image builds with any patching or configuring _at all_ (it has been deprecated with a warning for months already, and results are inconsistent)
This commit is contained in:
Ricardo Pardini
2023-05-01 15:15:42 +02:00
committed by igorpecovnik
parent f860106a2a
commit d890b418c7
11 changed files with 223 additions and 64 deletions

View File

@@ -99,6 +99,13 @@ function artifact_kernel_prepare_version() {
patches_hash="${hash_files}"
declare kernel_patches_hash_short="${patches_hash:0:${short_hash_size}}"
# detour: if CREATE_PATCHES=yes, then force "999999" (six-nines)
if [[ "${CREATE_PATCHES}" == "yes" ]]; then
display_alert "Forcing kernel patches hash to 999999" "due to CREATE_PATCHES=yes" "info"
patches_hash="999999 (CREATE_PATCHES=yes, unable to hash)"
kernel_patches_hash_short="999999"
fi
# get the .config hash... also userpatches...
declare kernel_config_source_filename="" # which actual .config was used?
prepare_kernel_config_core_or_userpatches
@@ -201,7 +208,10 @@ function artifact_kernel_prepare_version() {
function artifact_kernel_build_from_sources() {
compile_kernel
display_alert "Kernel build finished" "${artifact_version_reason}" "info"
if [[ "${ARTIFACT_WILL_NOT_BUILD}" != "yes" ]]; then # true if kernel-patch, kernel-config, etc.
display_alert "Kernel build finished" "${artifact_version_reason}" "info"
fi
}
function artifact_kernel_cli_adapter_pre_run() {
@@ -212,6 +222,16 @@ function artifact_kernel_cli_adapter_pre_run() {
}
function artifact_kernel_cli_adapter_config_prep() {
# Sanity check / cattle guard
# If KERNEL_CONFIGURE=yes, or CREATE_PATCHES=yes, user must have used the correct CLI commands, and only add those params.
if [[ "${KERNEL_CONFIGURE}" == "yes" && "${ARMBIAN_COMMAND}" != "kernel-config" ]]; then
exit_with_error "KERNEL_CONFIGURE=yes is not supported anymore. Please use the new 'kernel-config' CLI command. Current command: '${ARMBIAN_COMMAND}'"
fi
if [[ "${CREATE_PATCHES}" == "yes" && "${ARMBIAN_COMMAND}" != "kernel-patch" ]]; then
exit_with_error "CREATE_PATCHES=yes is not supported anymore. Please use the new 'kernel-patch' CLI command. Current command: '${ARMBIAN_COMMAND}'"
fi
use_board="yes" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
}

View File

@@ -127,17 +127,25 @@ function artifact_uboot_build_from_sources() {
LOG_SECTION="fetch_and_build_host_tools" do_with_logging fetch_and_build_host_tools
if [[ -n "${ATFSOURCE}" && "${ATFSOURCE}" != "none" ]]; then
LOG_SECTION="compile_atf" do_with_logging compile_atf
if [[ "${ARTIFACT_BUILD_INTERACTIVE:-"no"}" == "yes" ]]; then
display_alert "Running ATF build in interactive mode" "log file will be incomplete" "info"
compile_atf
if [[ "${CREATE_PATCHES_ATF:-"no"}" == "yes" ]]; then
return 0 # stop here, otherwise it would build u-boot below...
fi
else
LOG_SECTION="compile_atf" do_with_logging compile_atf
fi
fi
declare uboot_git_revision="not_determined_yet"
LOG_SECTION="uboot_prepare_git" do_with_logging_unless_user_terminal uboot_prepare_git
# Hack, if UBOOT_CONFIGURE=yes, don't run under logging manager. Emit a warning about it.
if [[ "${UBOOT_CONFIGURE:-"no"}" == "yes" ]]; then
display_alert "Warning" "UBOOT_CONFIGURE=yes, so we're not logging the build process of u-boot so it can be interactive." "wrn"
# Hack, if ARTIFACT_BUILD_INTERACTIVE=yes, don't run under logging manager. Emit a warning about it.
if [[ "${ARTIFACT_BUILD_INTERACTIVE:-"no"}" == "yes" ]]; then
display_alert "Running uboot build in interactive mode" "log file will be incomplete" "info"
compile_uboot
display_alert "Warning" "UBOOT_CONFIGURE=yes, so we've not logged the build process of u-boot so it could be interactive." "wrn"
else
LOG_SECTION="compile_uboot" do_with_logging compile_uboot
fi
@@ -151,6 +159,16 @@ function artifact_uboot_cli_adapter_pre_run() {
}
function artifact_uboot_cli_adapter_config_prep() {
# Sanity check / cattle guard
# If UBOOT_CONFIGURE=yes, or CREATE_PATCHES=yes, user must have used the correct CLI commands, and only add those params.
if [[ "${UBOOT_CONFIGURE}" == "yes" && ("${ARMBIAN_COMMAND}" != "uboot-config") ]]; then
exit_with_error "UBOOT_CONFIGURE=yes is not supported anymore. Please use the new 'uboot-config' CLI command. Current command: '${ARMBIAN_COMMAND}'"
fi
if [[ "${CREATE_PATCHES}" == "yes" && "${ARMBIAN_COMMAND}" != "uboot-patch" ]]; then
exit_with_error "CREATE_PATCHES=yes is not supported anymore. Please use the new 'uboot-patch' CLI command. Current command: '${ARMBIAN_COMMAND}'"
fi
use_board="yes" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
}

View File

@@ -226,6 +226,12 @@ function obtain_complete_artifact() {
# @TODO: if deploying to remote cache, force high compression, DEB_COMPRESS="xz"
artifact_build_from_sources # definitely will end up having its own logging sections
# For cases like CREATE_PATCHES=yes or KERNEL_CONFIGURE=yes, we wanna stop here. No artifact file will be created.
if [[ "${ARTIFACT_WILL_NOT_BUILD}" == "yes" ]]; then
display_alert "artifact" "ARTIFACT_WILL_NOT_BUILD is set, stopping after non-build." "debug"
return 0
fi
# pack the artifact to local cache (eg: for deb-tar)
LOG_SECTION="pack_artifact_to_local_cache" do_with_logging pack_artifact_to_local_cache
@@ -270,13 +276,7 @@ function build_artifact_for_image() {
# Make sure ORAS tooling is installed before starting.
run_tool_oras
# Detour: if building kernel, and KERNEL_CONFIGURE=yes, ignore artifact cache.
if [[ "${WHAT}" == "kernel" && "${KERNEL_CONFIGURE}" == "yes" ]]; then
display_alert "Ignoring artifact cache for kernel" "KERNEL_CONFIGURE=yes" "info"
ARTIFACT_IGNORE_CACHE="yes" obtain_complete_artifact
else
obtain_complete_artifact
fi
obtain_complete_artifact
return 0
}

View File

@@ -70,5 +70,10 @@ function cli_artifact_run() {
skip_unpack_if_found_in_caches="no"
fi
do_with_default_build obtain_complete_artifact # @TODO: < /dev/null -- but what about kernel configure?
if [[ "${ARTIFACT_BUILD_INTERACTIVE}" == "yes" ]]; then # Set by `kernel-config`, `kernel-patch`, `uboot-config`, `uboot-patch`, etc.
display_alert "Running artifact build in interactive mode" "log file will be incomplete" "info"
do_with_default_build obtain_complete_artifact
else
do_with_default_build obtain_complete_artifact < /dev/null
fi
}

View File

@@ -45,10 +45,13 @@ function armbian_register_commands() {
["rootfs"]="artifact"
["kernel"]="artifact"
["kernel-patch"]="artifact"
["kernel-config"]="artifact"
["u-boot"]="artifact"
["uboot"]="artifact"
["uboot-patch"]="artifact"
["atf-patch"]="artifact"
["uboot-config"]="artifact"
["firmware"]="artifact"
["firmware-full"]="artifact"
@@ -67,6 +70,9 @@ function armbian_register_commands() {
# common for all CLI-based artifact shortcuts
declare common_cli_artifact_vars=""
# common for interactive artifact shortcuts (configure, patch, etc)
declare common_cli_artifact_interactive_vars="ARTIFACT_WILL_NOT_BUILD='yes' ARTIFACT_BUILD_INTERACTIVE='yes' ARTIFACT_IGNORE_CACHE='yes'"
# Vars to be set for each command. Optional.
declare -g -A ARMBIAN_COMMANDS_TO_VARS_DICT=(
["docker-purge"]="DOCKER_SUBCMD='purge'"
@@ -81,11 +87,14 @@ function armbian_register_commands() {
# artifact shortcuts
["rootfs"]="WHAT='rootfs' ${common_cli_artifact_vars}"
["kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ARTIFACT_BUILD_INTERACTIVE='yes' ARTIFACT_IGNORE_CACHE='yes' ${common_cli_artifact_vars}"
["kernel"]="WHAT='kernel' ${common_cli_artifact_vars}"
["kernel-config"]="WHAT='kernel' KERNEL_CONFIGURE='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
["kernel-patch"]="WHAT='kernel' CREATE_PATCHES='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
["uboot"]="WHAT='uboot' ${common_cli_artifact_vars}"
["u-boot"]="WHAT='uboot' ${common_cli_artifact_vars}"
["uboot-config"]="WHAT='uboot' UBOOT_CONFIGURE='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
["uboot-patch"]="WHAT='uboot' CREATE_PATCHES='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
["atf-patch"]="WHAT='uboot' CREATE_PATCHES_ATF='yes' ${common_cli_artifact_interactive_vars} ${common_cli_artifact_vars}"
["firmware"]="WHAT='firmware' ${common_cli_artifact_vars}"
["firmware-full"]="WHAT='full_firmware' ${common_cli_artifact_vars}"

View File

@@ -59,7 +59,10 @@ compile_atf() {
advanced_patch "atf" "${ATFPATCHDIR}" "$BOARD" "$target_patchdir" "$BRANCH" "${LINUXFAMILY}-${BOARD}-${BRANCH}"
# create patch for manual source changes
[[ $CREATE_PATCHES == yes ]] && userpatch_create "atf"
if [[ $CREATE_PATCHES_ATF == yes ]]; then
userpatch_create "atf"
return 0
fi
# - "--no-warn-rwx-segment" is *required* for binutils 2.39 - see https://developer.trustedfirmware.org/T996
# - but *not supported* by 2.38, brilliant...

View File

@@ -54,6 +54,12 @@ function compile_kernel() {
return 0
fi
# Stop after creating patches.
if [[ "${CREATE_PATCHES}" == yes ]]; then
display_alert "Stopping after creating kernel patch" "" "cachehit"
return 0
fi
# patching worked, it's a good enough indication the git-bundle worked;
# let's clean up the git-bundle cache, since the git-bare cache is proven working.
LOG_SECTION="kernel_cleanup_bundle_artifacts" do_with_logging do_with_hooks kernel_cleanup_bundle_artifacts
@@ -69,6 +75,12 @@ function compile_kernel() {
kernel_config # has it's own logging sections inside
# Stop after configuring kernel.
if [[ "${KERNEL_CONFIGURE}" == yes ]]; then
display_alert "Stopping after configuring kernel" "" "cachehit"
return 0
fi
# build via make and package .debs; they're separate sub-steps
kernel_prepare_build_and_package # has it's own logging sections inside

View File

@@ -106,52 +106,123 @@ process_patch_file() {
return 0 # short-circuit above, avoid exiting with error
}
userpatch_create() {
display_alert "@TODO" "@TODO armbian-next" "warn"
# create commit to start from clean source
git add .
git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy"
function userpatch_create() {
declare patch_type="${1}"
declare -a common_git_params=(
"-c" "commit.gpgsign=false"
"-c" "user.name='${MAINTAINER}'"
"-c" "user.email='${MAINTAINERMAIL}'"
)
# export the commit as a patch
declare formatpatch_params=(
"-1" "HEAD" "--stdout"
"--unified=5" # force 5 lines of diff context
"--keep-subject" # do not add a prefix to the subject "[PATCH] "
'--signature' "'Created with Armbian build tools https://github.com/armbian/build'"
'--stat=120' # 'wider' stat output; default is 80
'--stat-graph-width=10' # shorten the diffgraph graph part, it's too long
"--zero-commit" # Output an all-zero hash in each patchs From header instead of the hash of the commit.
)
# if stdin is not a terminal, bail out
[[ -t 0 ]] || exit_with_error "patching: stdin is not a terminal"
[[ -t 1 ]] || exit_with_error "patching: stdout is not a terminal"
# Display a header with instructions about MAINTAINER and MAINTAINERMAIL
display_alert "Starting" "interactive patching process for ${patch_type}" "ext"
# create commit to start from clean source; don't fail.
display_alert "Creating commit to start from clean source" "" "info"
run_host_command_logged git "${common_git_params[@]}" add . "||" true
run_host_command_logged git "${common_git_params[@]}" commit -q -m "'Previous changes made by Armbian'" "||" true
display_alert "Patches will be created" "with the following maintainer information" "info"
display_alert "MAINTAINER (Real name): " "${MAINTAINER}" "info"
display_alert "MAINTAINERMAIL (Email): " "${MAINTAINERMAIL}" "info"
display_alert "If those are not correct, set them in your environment, command line, or config file and restart the process" "" ""
mkdir -p "${DEST}/patch"
local patch="$DEST/patch/$1-$LINUXFAMILY-$BRANCH.patch"
# apply previous user debug mode created patches
if [[ -f $patch ]]; then
display_alert "Applying existing $1 patch" "$patch" "wrn" && patch --batch --silent -p1 -N < "${patch}"
# read title of a patch in case Git is configured
if [[ -n $(git config user.email) ]]; then
COMMIT_MESSAGE=$(cat "${patch}" | grep Subject | sed -n -e '0,/PATCH/s/.*PATCH]//p' | xargs)
display_alert "Patch name extracted" "$COMMIT_MESSAGE" "wrn"
fi
fi
declare patch="${DEST}/patch/${patch_type}-${LINUXFAMILY}-${BRANCH}.patch"
# prompt to alter source
display_alert "Make your changes in this directory:" "$(pwd)" "wrn"
display_alert "Press <Enter> after you are done" "waiting" "wrn"
read -r < /dev/tty
tput cuu1
git add .
# create patch out of changes
if ! git diff-index --quiet --cached HEAD; then
# If Git is configured, create proper patch and ask for a name
if [[ -n $(git config user.email) ]]; then
display_alert "Add / change patch name" "$COMMIT_MESSAGE" "wrn"
read -e -p "Patch description: " -i "$COMMIT_MESSAGE" COMMIT_MESSAGE
[[ -z "$COMMIT_MESSAGE" ]] && COMMIT_MESSAGE="Patching something"
git commit -s -m "$COMMIT_MESSAGE"
git format-patch -1 HEAD --stdout --signature="Created with Armbian build tools https://github.com/armbian/build" > "${patch}"
PATCHFILE=$(git format-patch -1 HEAD)
rm $PATCHFILE # delete the actual file
# create a symlink to have a nice name ready
find $DEST/patch/ -type l -delete # delete any existing
ln -sf $patch $DEST/patch/$PATCHFILE
else
git diff --staged > "${patch}"
fi
display_alert "You will find your patch here:" "$patch" "info"
else
display_alert "No changes found, skipping patch creation" "" "wrn"
if [[ "${ARMBIAN_RUNNING_IN_CONTAINER}" == "yes" ]]; then
display_alert "You are running in a container" "Path shown above might not match host system, be aware." "wrn"
fi
# If the ${patch} file already exists, offer to apply it before continuing patching.
if [[ -f "${patch}" ]]; then
display_alert "A previously-created patch file already exists!" "${patch}" "wrn"
declare apply_patch
read -r -e -p "Do you want to apply it before continuing? [y/N] " apply_patch
if [[ "${apply_patch}" == "y" ]]; then
display_alert "Applying patch" "${patch}" "info"
run_host_command_logged git "${common_git_params[@]}" apply "${patch}" || display_alert "Patch failed to apply, continuing..." "${patch}" "wrn"
fi
fi
# Enter a loop, waiting for ENTER, then showing the git diff, and have the user confirm he is happy with patch
declare user_happy="no"
while [[ "${user_happy}" != "yes" ]]; do
display_alert "Press <ENTER> after you are done" "editing files in $(pwd)" "wrn"
# Wait for user to press ENTER
declare stop_patching
read -r -e -p "Press ENTER to show a preview of your patch, or type 'stop' to stop patching..." stop_patching
[[ "${stop_patching}" == "stop" ]] && exit_with_error "Aborting due to" "user request"
# Detect if there are any changes done to the working tree
declare -i changes_in_working_tree
changes_in_working_tree=$(git "${common_git_params[@]}" status --porcelain | wc -l)
if [[ ${changes_in_working_tree} -lt 1 ]]; then
display_alert "No changes detected!" "No changes in the working tree, please edit files and try again" "wrn"
continue # no changes, loop again
fi
display_alert "OK, here's how your diff looks like" "showing patch diff" "info"
git "${common_git_params[@]}" diff | run_tool_batcat --file-name "${patch}" -
# Prompt the user if he is happy with the patch
display_alert "Are you happy with this patch?" "Type 'yes' to accept, 'stop' to stop patching, or anything else to keep patching" "wrn"
# Wait for user to type yes or no
read -r -e -p "Are you happy with the diff above? Type 'y' or 'yes' to accept, 'stop' to stop patching, anything else to keep patching: " -i "" user_happy
declare first_uppercase_character_of_user_happy="${user_happy:0:1}"
first_uppercase_character_of_user_happy="${first_uppercase_character_of_user_happy^^}"
[[ "${first_uppercase_character_of_user_happy}" == "Y" ]] && break
[[ "${first_uppercase_character_of_user_happy}" == "S" ]] && exit_with_error "Aborting due to user request"
display_alert "Not happy? No problem!" "just keep on editing the files..." "wrn"
done
display_alert "OK, user is happy with diff" "proceeding with patch creation" "ext"
run_host_command_logged git add .
# create patch out of changes
if ! git "${common_git_params[@]}" diff-index --quiet --cached HEAD; then
# Default the patch_commit_message.
# Get a list of all the filenames in the git diff into a bash array...
declare -a changed_filenames=($(git "${common_git_params[@]}" diff-index --cached --name-only HEAD))
display_alert "Names of the changed files" "${changed_filenames[*]@Q}" "info"
declare patch_commit_message="Patching ${patch_type} ${LINUXFAMILY} files ${changed_filenames[*]@Q}"
# If Git is configured, create proper patch and ask for a name
display_alert "Add / change patch name" "${patch_commit_message}" "wrn"
read -e -p "Patch Subject: " -i "${patch_commit_message}" patch_commit_message
[[ -z "${patch_commit_message}" ]] && patch_commit_message="Patching something unknown and mysterious"
run_host_command_logged git "${common_git_params[@]}" commit -s -m "'${patch_commit_message}'"
run_host_command_logged git "${common_git_params[@]}" format-patch "${formatpatch_params[@]}" ">" "${patch}"
display_alert "You will find your patch here:" "${patch}" "info"
run_tool_batcat --file-name "${patch}" "${patch}"
display_alert "You will find your patch here:" "${patch}" "info"
display_alert "Now you can manually move the produced patch to your userpatches or core patches to have it included in the next build" "${patch}" "ext"
else
display_alert "No changes found, skipping patch creation" "" "err"
fi
git reset --soft HEAD~
for i in {3..1..1}; do echo -n "$i." && sleep 1; done
}

View File

@@ -43,7 +43,10 @@ function compile_uboot_target() {
do_with_hooks uboot_main_patching_python
# create patch for manual source changes
[[ $CREATE_PATCHES == yes ]] && userpatch_create "u-boot"
if [[ $CREATE_PATCHES == yes ]]; then
userpatch_create "u-boot"
return 0 # exit after this.
fi
# atftempdir comes from atf.sh's compile_atf()
if [[ -n $ATFSOURCE && -d "${atftempdir}" ]]; then
@@ -150,6 +153,7 @@ function compile_uboot_target() {
display_alert "Exporting saved config" "UBOOT_CONFIGURE=yes; experimental" "warn"
run_host_command_logged make savedefconfig
run_host_command_logged cp -v defconfig "${DEST}/defconfig-uboot-${BOARD}-${BRANCH}"
return 0 # exit after this
fi
# workaround when two compilers are needed
@@ -345,6 +349,11 @@ function compile_uboot() {
display_alert "Extensions: custom uboot built by extension" "not building regular uboot" "debug"
fi
if [[ "${ARTIFACT_WILL_NOT_BUILD:-"no"}" == "yes" ]]; then
display_alert "Extensions: artifact will not build" "not building regular uboot" "debug"
return 0
fi
display_alert "Preparing u-boot general packaging" "${version} ${target_make}"
# set up postinstall script # @todo: extract into a tinkerboard extension

View File

@@ -19,7 +19,6 @@
# find_toolchain
# advanced_patch
# process_patch_file
# userpatch_create
# overlayfs_wrapper
grab_version() {

View File

@@ -11,9 +11,22 @@
function full_build_packages_rootfs_and_image() {
error_if_kernel_only_set
# Detour, warn the user about KERNEL_CONFIGURE=yes if it is set.
# Detour, stop if KERNEL_CONFIGURE=yes
if [[ "${KERNEL_CONFIGURE}" == "yes" ]]; then
display_alert "KERNEL_CONFIGURE=yes during image build is deprecated." "It still works, but please prefer the new way. First, run './compile.sh BOARD=${BOARD} BRANCH=${BRANCH} kernel-config'; then commit your changes; then build the image as normal. This workflow ensures consistent hashing results." "wrn"
display_alert "KERNEL_CONFIGURE=yes during image build is not supported anymore." "First, run './compile.sh BOARD=${BOARD} BRANCH=${BRANCH} kernel-config'; then commit your changes; then build the image as normal. This workflow ensures consistent hashing results." "wrn"
exit_with_error "KERNEL_CONFIGURE=yes during image build is not supported anymore. Please use the new 'kernel-config' CLI command."
fi
# Detour, stop if UBOOT_CONFIGURE=yes
if [[ "${UBOOT_CONFIGURE}" == "yes" ]]; then
display_alert "UBOOT_CONFIGURE=yes during image build is not supported anymore." "First, run './compile.sh BOARD=${BOARD} BRANCH=${BRANCH} uboot-config'; then commit your changes; then build the image as normal. This workflow ensures consistent hashing results." "wrn"
exit_with_error "UBOOT_CONFIGURE=yes during image build is not supported anymore. Please use the new 'uboot-config' CLI command."
fi
# Detour, stop if CREATE_PATCHES=yes.
if [[ "${CREATE_PATCHES}" == "yes" || "${CREATE_PATCHES_ATF}" == "yes" ]]; then
display_alert "CREATE_PATCHES=yes during image build is not supported anymore." "First, run './compile.sh BOARD=${BOARD} BRANCH=${BRANCH} kernel-patch'; then move the patch to the correct place and commit your changes; then build the image as normal. This workflow ensures consistent hashing results." "wrn"
exit_with_error "CREATE_PATCHES=yes during image build is not supported anymore. Please use the new 'kernel-patch' / 'uboot-patch' / 'atf-patch' CLI commands."
fi
main_default_build_packages # has its own logging sections # requires aggregation