JDM170/build
1
0
Fork 0
mirror of https://github.com/armbian/build synced 2025-09-24 19:47:06 +07:00
Code Issues Packages Projects Releases Wiki Activity

armbian-next: post_build_image: run hook first, then write to CARD_DEVICE, then compress, in that order

Browse Source 
- compress *all* present output images, not only .img
- remove 7zip compression & hostdep
- squash a few shortcircuits (yeah, this late in the game...)
This commit is contained in:
Ricardo Pardini
2023-02-12 14:00:04 +01:00
parent 3d8ee5ce38
commit a2304f28b0
9 changed files with 52 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
extensions/image-output-ovf.sh
View File

@@ -113,4 +113,5 @@ function post_build_image__920_create_ovf() {
display_alert "Done, cleaning up" "${EXTENSION}" "info" display_alert "Done, cleaning up" "${EXTENSION}" "info"
rm -rf "${full_vmware_dirname}" rm -rf "${full_vmware_dirname}"
return 0
} }

2
extensions/image-output-qcow2.sh
View File

@@ -5,6 +5,7 @@ function add_host_dependencies__qcow2_host_deps() {
function post_build_image__900_convert_to_qcow2_img() { function post_build_image__900_convert_to_qcow2_img() {
[[ "${SKIP_QCOW2}" == "yes" ]] && return 0 [[ "${SKIP_QCOW2}" == "yes" ]] && return 0
[[ -z $version ]] && exit_with_error "version is not set"
display_alert "Converting image to qcow2" "${EXTENSION}" "info" display_alert "Converting image to qcow2" "${EXTENSION}" "info"
export QCOW2_IMAGE_FILE="${DESTIMG}/${version}.img.qcow2" export QCOW2_IMAGE_FILE="${DESTIMG}/${version}.img.qcow2"
run_host_command_logged qemu-img convert -f raw -O qcow2 "${DESTIMG}/${version}.img" "${QCOW2_IMAGE_FILE}" run_host_command_logged qemu-img convert -f raw -O qcow2 "${DESTIMG}/${version}.img" "${QCOW2_IMAGE_FILE}"
@@ -17,4 +18,5 @@ function post_build_image__900_convert_to_qcow2_img() {
display_alert "Discarding original .img image after" "conversion to qcow2" "info" display_alert "Discarding original .img image after" "conversion to qcow2" "info"
run_host_command_logged rm -vf "${DESTIMG}/${version}.img" "${DESTIMG}/${version}.img.txt" run_host_command_logged rm -vf "${DESTIMG}/${version}.img" "${DESTIMG}/${version}.img.txt"
fi fi
return 0
} }

1
extensions/image-output-utm.sh
View File

@@ -179,4 +179,5 @@ function post_build_image__920_create_utm_plist() {
display_alert "Done, cleaning up" "${EXTENSION}" "info" display_alert "Done, cleaning up" "${EXTENSION}" "info"
rm -rf "${full_utm_dirname}" rm -rf "${full_utm_dirname}"
return 0
} }

12
lib/functions/host/prepare-host.sh
View File

@@ -264,12 +264,12 @@ function adaptative_prepare_host_dependencies() {
zlib1g-dev zlib1g-dev
# by-category below # by-category below
file tree expect # logging utilities; expect is needed for 'unbuffer' command file tree expect # logging utilities; expect is needed for 'unbuffer' command
colorized-logs # for ansi2html, ansi2txt, pipetty colorized-logs # for ansi2html, ansi2txt, pipetty
unzip zip p7zip-full pigz pixz pbzip2 lzop zstd # compressors et al unzip zip pigz pixz pbzip2 lzop zstd # compressors et al
parted gdisk fdisk # partition tools @TODO why so many? parted gdisk fdisk # partition tools @TODO why so many?
aria2 curl wget axel # downloaders et al aria2 curl wget axel # downloaders et al
parallel # do things in parallel (used for fast md5 hashing in initrd cache) parallel # do things in parallel (used for fast md5 hashing in initrd cache)
) )
# @TODO: distcc -- handle in extension? # @TODO: distcc -- handle in extension?

59
lib/functions/image/compress-checksum.sh
View File

@@ -1,44 +1,41 @@
function image_compress_and_checksum() { function output_images_compress_and_checksum() {
[[ -n $SEND_TO_SERVER ]] && return 0 [[ -n $SEND_TO_SERVER ]] && return 0
# check that 'version' is set # check that 'version' is set
[[ -z $version ]] && exit_with_error "version is not set" [[ -z $version ]] && exit_with_error "version is not set"
# compression_type: declared in outer scope # compression_type: declared in outer scope
if [[ $COMPRESS_OUTPUTIMAGE == *gz* ]]; then declare prefix_images="${1}"
display_alert "Compressing" "${DESTIMG}/${version}.img.gz" "info" # find all files that match prefix_images
pigz -3 < "$DESTIMG/${version}".img > "$DESTIMG/${version}".img.gz declare -a images=("${prefix_images}"*)
compression_type=".gz" # if no files match prefix_images, exit
if [[ ${#images[@]} -eq 0 ]]; then
display_alert "No files to compress and checksum" "no images will be compressed" "wrn"
return 0
fi fi
if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then # loop over images
# @TODO: rpardini: I'd just move to zstd and be done with it. It does it right. for uncompressed_file in "${images[@]}"; do
display_alert "Compressing" "${DESTIMG}/${version}.img.xz" "info" # if image is a symlink, skip it
declare -i available_cpu [[ -L "${uncompressed_file}" ]] && continue
available_cpu=$(grep -c 'processor' /proc/cpuinfo) # if image is not a file, skip it
[[ ${available_cpu} -gt 16 ]] && available_cpu=16 # using more cpu cores for compressing is pointless [[ ! -f "${uncompressed_file}" ]] && continue
pixz -7 -p ${available_cpu} -f $((available_cpu + 2)) < "$DESTIMG/${version}".img > "${DESTIMG}/${version}".img.xz
compression_type=".xz"
fi
if [[ $COMPRESS_OUTPUTIMAGE == *img* || $COMPRESS_OUTPUTIMAGE == *7z* ]]; then # get just the filename, sans path
compression_type="" declare uncompressed_file_basename
fi uncompressed_file_basename=$(basename "${uncompressed_file}")
if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then
cd "${DESTIMG}" || exit_with_error "Could not cd to ${DESTIMG}" display_alert "Compressing with pixz" "${uncompressed_file_basename}.xz" "info"
display_alert "SHA256 calculating" "${version}.img${compression_type}" "info" pixz -7 "${uncompressed_file}" # "If pixz is provided with input but no output, it will delete the input"
sha256sum -b "${version}.img${compression_type}" > "${version}.img${compression_type}".sha compression_type=".xz"
fi fi
fingerprint_image "${DESTIMG}/${version}.img${compression_type}.txt" "${version}" if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then
display_alert "SHA256 calculating" "${uncompressed_file_basename}${compression_type}" "info"
sha256sum -b "${uncompressed_file}${compression_type}" > "${uncompressed_file}${compression_type}".sha
fi
done
if [[ $COMPRESS_OUTPUTIMAGE == *7z* ]]; then
display_alert "Untested code path, bumpy road ahead" "7z compression" "wrn"
display_alert "Compressing" "${DESTIMG}/${version}.7z" "info"
7za a -t7z -bd -m0=lzma2 -mx=3 -mfb=64 -md=32m -ms=on "${DESTIMG}/${version}".7z "${version}".key "${version}".img*
find "${DESTIMG}"/ -type
f \( -name "${version}.img" -o -name "${version}.img.asc" -o -name "${version}.img.txt" -o -name "${version}.img.sha" \) -print0 |
xargs -0 rm
fi
} }

6
lib/functions/image/fingerprint.sh
View File

@@ -3,11 +3,11 @@
# fingerprint_image <out_txt_file> [image_filename] # fingerprint_image <out_txt_file> [image_filename]
# Saving build summary to the image # Saving build summary to the image
#-------------------------------------------------------------------------------------------------------------------------------- #--------------------------------------------------------------------------------------------------------------------------------
fingerprint_image() { function fingerprint_image() {
cat <<- EOF > "${1}" cat <<- EOF > "${1}"
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
Title: ${VENDOR} $REVISION ${BOARD^} $BRANCH Title: ${VENDOR} $REVISION ${BOARD^} $BRANCH
Kernel: Linux $VER Kernel: Linux ${IMAGE_INSTALLED_KERNEL_VERSION}
Build date: $(date +'%d.%m.%Y') Build date: $(date +'%d.%m.%Y')
Builder rev: ${BUILD_REPOSITORY_COMMIT} Builder rev: ${BUILD_REPOSITORY_COMMIT}
Maintainer: $MAINTAINER <$MAINTAINERMAIL> Maintainer: $MAINTAINER <$MAINTAINERMAIL>
@@ -15,7 +15,7 @@ fingerprint_image() {
Sources: https://github.com/armbian/ Sources: https://github.com/armbian/
Support: https://forum.armbian.com/ Support: https://forum.armbian.com/
Changelog: https://www.armbian.com/logbook/ Changelog: https://www.armbian.com/logbook/
Documantation: https://docs.armbian.com/ Documentation: https://docs.armbian.com/
EOF EOF
if [ -n "$2" ]; then if [ -n "$2" ]; then

21
lib/functions/image/rootfs-to-image.sh
View File

@@ -105,9 +105,6 @@ function create_image_from_sdcard_rootfs() {
# custom post_build_image_modify hook to run before fingerprinting and compression # custom post_build_image_modify hook to run before fingerprinting and compression
[[ $(type -t post_build_image_modify) == function ]] && display_alert "Custom Hook Detected" "post_build_image_modify" "info" && post_build_image_modify "${DESTIMG}/${version}.img" [[ $(type -t post_build_image_modify) == function ]] && display_alert "Custom Hook Detected" "post_build_image_modify" "info" && post_build_image_modify "${DESTIMG}/${version}.img"
declare compression_type # set by image_compress_and_checksum
image_compress_and_checksum
# Previously, post_build_image passed the .img path as an argument to the hook. Now its an ENV var. # Previously, post_build_image passed the .img path as an argument to the hook. Now its an ENV var.
export FINAL_IMAGE_FILE="${DESTIMG}/${version}.img" export FINAL_IMAGE_FILE="${DESTIMG}/${version}.img"
call_extension_method "post_build_image" <<- 'POST_BUILD_IMAGE' call_extension_method "post_build_image" <<- 'POST_BUILD_IMAGE'
@@ -118,23 +115,23 @@ function create_image_from_sdcard_rootfs() {
It is the last possible chance to modify `$CARD_DEVICE`. It is the last possible chance to modify `$CARD_DEVICE`.
POST_BUILD_IMAGE POST_BUILD_IMAGE
# If we compressed the image, get rid of the original, and leave only the compressed one. # Before compressing or moving, write it to SD card if such was requested and image was produced.
[[ -n $compression_type ]] && rm -f "${DESTIMG}/${version}.img" if [[ -f "${DESTIMG}/${version}.img" ]]; then
if [[ -n $compression_type ]]; then display_alert "Done building" "${version}.img" "info"
run_host_command_logged rm -v "${DESTIMG}/${version}.img" fingerprint_image "${DESTIMG}/${version}.img.txt" "${version}"
# write image to SD card
write_image_to_device "${DESTIMG}/${version}.img" "${CARD_DEVICE}"
fi fi
declare compression_type # set by image_compress_and_checksum
output_images_compress_and_checksum "${DESTIMG}/${version}" # this compressed on-disk, and removes the originals.
# Move all files matching the prefix from source to dest. Custom hooks might generate more than one img. # Move all files matching the prefix from source to dest. Custom hooks might generate more than one img.
declare source_dir="${DESTIMG}" declare source_dir="${DESTIMG}"
declare destination_dir="${FINALDEST}" declare destination_dir="${FINALDEST}"
declare source_files_prefix="${version}" declare source_files_prefix="${version}"
move_images_to_final_destination move_images_to_final_destination
display_alert "Done building" "${FINALDEST}/${version}.img" "info" # A bit predicting the future, since it's still in DESTIMG at this point.
# write image to SD card
write_image_to_device "${FINALDEST}/${version}.img" "${CARD_DEVICE}"
return 0 return 0
} }

4
lib/functions/image/write-device.sh
View File

@@ -20,11 +20,11 @@ function write_image_to_device() {
# write to SD card # write to SD card
pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "write_device") dd" "${image_file}" | dd "of=${device}" bs=1M iflag=fullblock oflag=direct status=none pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "write_device") dd" "${image_file}" | dd "of=${device}" bs=1M iflag=fullblock oflag=direct status=none
call_extension_method "post_write_sdcard" <<- 'POST_BUILD_IMAGE' call_extension_method "post_write_sdcard" <<- 'POST_WRITE_SDCARD'
*run after writing img to sdcard* *run after writing img to sdcard*
After the image is written to `${device}`, but before verifying it. After the image is written to `${device}`, but before verifying it.
You can still set SKIP_VERIFY=yes to skip verification. You can still set SKIP_VERIFY=yes to skip verification.
POST_BUILD_IMAGE POST_WRITE_SDCARD
if [[ "${SKIP_VERIFY}" != "yes" ]]; then if [[ "${SKIP_VERIFY}" != "yes" ]]; then
# read and compare # read and compare

2
lib/functions/main/config-prepare.sh
View File

@@ -169,8 +169,6 @@ function config_pre_main() {
function config_post_main() { function config_post_main() {
if [[ $COMPRESS_OUTPUTIMAGE == "" || $COMPRESS_OUTPUTIMAGE == no ]]; then if [[ $COMPRESS_OUTPUTIMAGE == "" || $COMPRESS_OUTPUTIMAGE == no ]]; then
COMPRESS_OUTPUTIMAGE="sha,img" COMPRESS_OUTPUTIMAGE="sha,img"
elif [[ $COMPRESS_OUTPUTIMAGE == yes ]]; then
COMPRESS_OUTPUTIMAGE="sha,7z"
fi fi
if [[ "$BETA" == "yes" ]]; then if [[ "$BETA" == "yes" ]]; then