From a2304f28b09800bf7e991d2913e529852749a927 Mon Sep 17 00:00:00 2001 From: Ricardo Pardini Date: Sun, 12 Feb 2023 14:00:04 +0100 Subject: [PATCH] armbian-next: post_build_image: run hook first, then write to CARD_DEVICE, then compress, in that order - compress *all* present output images, not only .img - remove 7zip compression & hostdep - squash a few shortcircuits (yeah, this late in the game...) --- extensions/image-output-ovf.sh | 1 + extensions/image-output-qcow2.sh | 2 + extensions/image-output-utm.sh | 1 + lib/functions/host/prepare-host.sh | 12 ++--- lib/functions/image/compress-checksum.sh | 59 +++++++++++------------- lib/functions/image/fingerprint.sh | 6 +-- lib/functions/image/rootfs-to-image.sh | 21 ++++----- lib/functions/image/write-device.sh | 4 +- lib/functions/main/config-prepare.sh | 2 - 9 files changed, 52 insertions(+), 56 deletions(-) diff --git a/extensions/image-output-ovf.sh b/extensions/image-output-ovf.sh index 50fd64383..8fbaeefe3 100644 --- a/extensions/image-output-ovf.sh +++ b/extensions/image-output-ovf.sh @@ -113,4 +113,5 @@ function post_build_image__920_create_ovf() { display_alert "Done, cleaning up" "${EXTENSION}" "info" rm -rf "${full_vmware_dirname}" + return 0 } diff --git a/extensions/image-output-qcow2.sh b/extensions/image-output-qcow2.sh index 72ade0276..60fd5b61b 100644 --- a/extensions/image-output-qcow2.sh +++ b/extensions/image-output-qcow2.sh @@ -5,6 +5,7 @@ function add_host_dependencies__qcow2_host_deps() { function post_build_image__900_convert_to_qcow2_img() { [[ "${SKIP_QCOW2}" == "yes" ]] && return 0 + [[ -z $version ]] && exit_with_error "version is not set" display_alert "Converting image to qcow2" "${EXTENSION}" "info" export QCOW2_IMAGE_FILE="${DESTIMG}/${version}.img.qcow2" run_host_command_logged qemu-img convert -f raw -O qcow2 "${DESTIMG}/${version}.img" "${QCOW2_IMAGE_FILE}" @@ -17,4 +18,5 @@ function post_build_image__900_convert_to_qcow2_img() { display_alert "Discarding original .img image after" "conversion to qcow2" "info" run_host_command_logged rm -vf "${DESTIMG}/${version}.img" "${DESTIMG}/${version}.img.txt" fi + return 0 } diff --git a/extensions/image-output-utm.sh b/extensions/image-output-utm.sh index 4493d226f..efa7b239d 100644 --- a/extensions/image-output-utm.sh +++ b/extensions/image-output-utm.sh @@ -179,4 +179,5 @@ function post_build_image__920_create_utm_plist() { display_alert "Done, cleaning up" "${EXTENSION}" "info" rm -rf "${full_utm_dirname}" + return 0 } diff --git a/lib/functions/host/prepare-host.sh b/lib/functions/host/prepare-host.sh index 2e3f29883..32afc71fe 100644 --- a/lib/functions/host/prepare-host.sh +++ b/lib/functions/host/prepare-host.sh @@ -264,12 +264,12 @@ function adaptative_prepare_host_dependencies() { zlib1g-dev # by-category below - file tree expect # logging utilities; expect is needed for 'unbuffer' command - colorized-logs # for ansi2html, ansi2txt, pipetty - unzip zip p7zip-full pigz pixz pbzip2 lzop zstd # compressors et al - parted gdisk fdisk # partition tools @TODO why so many? - aria2 curl wget axel # downloaders et al - parallel # do things in parallel (used for fast md5 hashing in initrd cache) + file tree expect # logging utilities; expect is needed for 'unbuffer' command + colorized-logs # for ansi2html, ansi2txt, pipetty + unzip zip pigz pixz pbzip2 lzop zstd # compressors et al + parted gdisk fdisk # partition tools @TODO why so many? + aria2 curl wget axel # downloaders et al + parallel # do things in parallel (used for fast md5 hashing in initrd cache) ) # @TODO: distcc -- handle in extension? diff --git a/lib/functions/image/compress-checksum.sh b/lib/functions/image/compress-checksum.sh index 489aac5aa..de7fed50d 100644 --- a/lib/functions/image/compress-checksum.sh +++ b/lib/functions/image/compress-checksum.sh @@ -1,44 +1,41 @@ -function image_compress_and_checksum() { +function output_images_compress_and_checksum() { [[ -n $SEND_TO_SERVER ]] && return 0 # check that 'version' is set [[ -z $version ]] && exit_with_error "version is not set" # compression_type: declared in outer scope - if [[ $COMPRESS_OUTPUTIMAGE == *gz* ]]; then - display_alert "Compressing" "${DESTIMG}/${version}.img.gz" "info" - pigz -3 < "$DESTIMG/${version}".img > "$DESTIMG/${version}".img.gz - compression_type=".gz" + declare prefix_images="${1}" + # find all files that match prefix_images + declare -a images=("${prefix_images}"*) + # if no files match prefix_images, exit + if [[ ${#images[@]} -eq 0 ]]; then + display_alert "No files to compress and checksum" "no images will be compressed" "wrn" + return 0 fi - if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then - # @TODO: rpardini: I'd just move to zstd and be done with it. It does it right. - display_alert "Compressing" "${DESTIMG}/${version}.img.xz" "info" - declare -i available_cpu - available_cpu=$(grep -c 'processor' /proc/cpuinfo) - [[ ${available_cpu} -gt 16 ]] && available_cpu=16 # using more cpu cores for compressing is pointless - pixz -7 -p ${available_cpu} -f $((available_cpu + 2)) < "$DESTIMG/${version}".img > "${DESTIMG}/${version}".img.xz - compression_type=".xz" - fi + # loop over images + for uncompressed_file in "${images[@]}"; do + # if image is a symlink, skip it + [[ -L "${uncompressed_file}" ]] && continue + # if image is not a file, skip it + [[ ! -f "${uncompressed_file}" ]] && continue + + # get just the filename, sans path + declare uncompressed_file_basename + uncompressed_file_basename=$(basename "${uncompressed_file}") - if [[ $COMPRESS_OUTPUTIMAGE == *img* || $COMPRESS_OUTPUTIMAGE == *7z* ]]; then - compression_type="" - fi + if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then + display_alert "Compressing with pixz" "${uncompressed_file_basename}.xz" "info" + pixz -7 "${uncompressed_file}" # "If pixz is provided with input but no output, it will delete the input" + compression_type=".xz" + fi - if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then - cd "${DESTIMG}" || exit_with_error "Could not cd to ${DESTIMG}" - display_alert "SHA256 calculating" "${version}.img${compression_type}" "info" - sha256sum -b "${version}.img${compression_type}" > "${version}.img${compression_type}".sha - fi + if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then + display_alert "SHA256 calculating" "${uncompressed_file_basename}${compression_type}" "info" + sha256sum -b "${uncompressed_file}${compression_type}" > "${uncompressed_file}${compression_type}".sha + fi - fingerprint_image "${DESTIMG}/${version}.img${compression_type}.txt" "${version}" + done - if [[ $COMPRESS_OUTPUTIMAGE == *7z* ]]; then - display_alert "Untested code path, bumpy road ahead" "7z compression" "wrn" - display_alert "Compressing" "${DESTIMG}/${version}.7z" "info" - 7za a -t7z -bd -m0=lzma2 -mx=3 -mfb=64 -md=32m -ms=on "${DESTIMG}/${version}".7z "${version}".key "${version}".img* - find "${DESTIMG}"/ -type - f \( -name "${version}.img" -o -name "${version}.img.asc" -o -name "${version}.img.txt" -o -name "${version}.img.sha" \) -print0 | - xargs -0 rm - fi } diff --git a/lib/functions/image/fingerprint.sh b/lib/functions/image/fingerprint.sh index cec9eb42c..4720dc498 100644 --- a/lib/functions/image/fingerprint.sh +++ b/lib/functions/image/fingerprint.sh @@ -3,11 +3,11 @@ # fingerprint_image [image_filename] # Saving build summary to the image #-------------------------------------------------------------------------------------------------------------------------------- -fingerprint_image() { +function fingerprint_image() { cat <<- EOF > "${1}" -------------------------------------------------------------------------------- Title: ${VENDOR} $REVISION ${BOARD^} $BRANCH - Kernel: Linux $VER + Kernel: Linux ${IMAGE_INSTALLED_KERNEL_VERSION} Build date: $(date +'%d.%m.%Y') Builder rev: ${BUILD_REPOSITORY_COMMIT} Maintainer: $MAINTAINER <$MAINTAINERMAIL> @@ -15,7 +15,7 @@ fingerprint_image() { Sources: https://github.com/armbian/ Support: https://forum.armbian.com/ Changelog: https://www.armbian.com/logbook/ - Documantation: https://docs.armbian.com/ + Documentation: https://docs.armbian.com/ EOF if [ -n "$2" ]; then diff --git a/lib/functions/image/rootfs-to-image.sh b/lib/functions/image/rootfs-to-image.sh index 1734897ba..57a67304e 100644 --- a/lib/functions/image/rootfs-to-image.sh +++ b/lib/functions/image/rootfs-to-image.sh @@ -105,9 +105,6 @@ function create_image_from_sdcard_rootfs() { # custom post_build_image_modify hook to run before fingerprinting and compression [[ $(type -t post_build_image_modify) == function ]] && display_alert "Custom Hook Detected" "post_build_image_modify" "info" && post_build_image_modify "${DESTIMG}/${version}.img" - declare compression_type # set by image_compress_and_checksum - image_compress_and_checksum - # Previously, post_build_image passed the .img path as an argument to the hook. Now its an ENV var. export FINAL_IMAGE_FILE="${DESTIMG}/${version}.img" call_extension_method "post_build_image" <<- 'POST_BUILD_IMAGE' @@ -118,23 +115,23 @@ function create_image_from_sdcard_rootfs() { It is the last possible chance to modify `$CARD_DEVICE`. POST_BUILD_IMAGE - # If we compressed the image, get rid of the original, and leave only the compressed one. - [[ -n $compression_type ]] && rm -f "${DESTIMG}/${version}.img" - if [[ -n $compression_type ]]; then - run_host_command_logged rm -v "${DESTIMG}/${version}.img" + # Before compressing or moving, write it to SD card if such was requested and image was produced. + if [[ -f "${DESTIMG}/${version}.img" ]]; then + display_alert "Done building" "${version}.img" "info" + fingerprint_image "${DESTIMG}/${version}.img.txt" "${version}" + # write image to SD card + write_image_to_device "${DESTIMG}/${version}.img" "${CARD_DEVICE}" fi + declare compression_type # set by image_compress_and_checksum + output_images_compress_and_checksum "${DESTIMG}/${version}" # this compressed on-disk, and removes the originals. + # Move all files matching the prefix from source to dest. Custom hooks might generate more than one img. declare source_dir="${DESTIMG}" declare destination_dir="${FINALDEST}" declare source_files_prefix="${version}" move_images_to_final_destination - display_alert "Done building" "${FINALDEST}/${version}.img" "info" # A bit predicting the future, since it's still in DESTIMG at this point. - - # write image to SD card - write_image_to_device "${FINALDEST}/${version}.img" "${CARD_DEVICE}" - return 0 } diff --git a/lib/functions/image/write-device.sh b/lib/functions/image/write-device.sh index b4f5165f9..6982a4f43 100644 --- a/lib/functions/image/write-device.sh +++ b/lib/functions/image/write-device.sh @@ -20,11 +20,11 @@ function write_image_to_device() { # write to SD card pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "write_device") dd" "${image_file}" | dd "of=${device}" bs=1M iflag=fullblock oflag=direct status=none - call_extension_method "post_write_sdcard" <<- 'POST_BUILD_IMAGE' + call_extension_method "post_write_sdcard" <<- 'POST_WRITE_SDCARD' *run after writing img to sdcard* After the image is written to `${device}`, but before verifying it. You can still set SKIP_VERIFY=yes to skip verification. - POST_BUILD_IMAGE + POST_WRITE_SDCARD if [[ "${SKIP_VERIFY}" != "yes" ]]; then # read and compare diff --git a/lib/functions/main/config-prepare.sh b/lib/functions/main/config-prepare.sh index 280fe14cc..a8edab158 100644 --- a/lib/functions/main/config-prepare.sh +++ b/lib/functions/main/config-prepare.sh @@ -169,8 +169,6 @@ function config_pre_main() { function config_post_main() { if [[ $COMPRESS_OUTPUTIMAGE == "" || $COMPRESS_OUTPUTIMAGE == no ]]; then COMPRESS_OUTPUTIMAGE="sha,img" - elif [[ $COMPRESS_OUTPUTIMAGE == yes ]]; then - COMPRESS_OUTPUTIMAGE="sha,7z" fi if [[ "$BETA" == "yes" ]]; then