From fdd305e52f0f3aa93235a1b5fba7dc5ef85eb6f3 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Wed, 23 Dec 2020 18:19:30 +0100 Subject: [PATCH] Several backend fixes and improvemetens Affected: - repository old packages purging - multiple image making --- lib/debootstrap.sh | 63 ++++++++++++++++++++++---------------------- lib/general.sh | 38 +++++++++++++------------- lib/image-helpers.sh | 4 +-- 3 files changed, 52 insertions(+), 53 deletions(-) diff --git a/lib/debootstrap.sh b/lib/debootstrap.sh index a9af5d609..b5aea329d 100644 --- a/lib/debootstrap.sh +++ b/lib/debootstrap.sh @@ -649,6 +649,18 @@ create_image() mkdir -p $DESTIMG mv ${SDCARD}.raw $DESTIMG/${version}.img + FINALDEST=$DEST/images + + if [[ $BUILD_ALL == yes ]]; then + if [[ "$BETA" == yes ]]; then + FINALDEST=$DEST/images/"${BOARD}"/nightly + else + FINALDEST=$DEST/images/"${BOARD}"/archive + fi + install -d -o nobody -g nogroup -m 775 ${FINALDEST}/${BOARD}/{archive,nightly} + fi + + if [[ -z $SEND_TO_SERVER ]]; then if [[ $COMPRESS_OUTPUTIMAGE == "" || $COMPRESS_OUTPUTIMAGE == no ]]; then @@ -658,85 +670,74 @@ create_image() fi if [[ $COMPRESS_OUTPUTIMAGE == *gz* ]]; then - display_alert "Compressing" "$DEST/images/${version}.img.gz" "info" - pigz -3 < $DESTIMG/${version}.img > $DEST/images/${version}.img.gz + display_alert "Compressing" "${FINALDEST}/${version}.img.gz" "info" + pigz -3 < $DESTIMG/${version}.img > ${FINALDEST}/${version}.img.gz compression_type=".gz" fi if [[ $COMPRESS_OUTPUTIMAGE == *xz* ]]; then - display_alert "Compressing" "$DEST/images/${version}.img.xz" "info" - pixz -9 < $DESTIMG/${version}.img > $DEST/images/${version}.img.xz + display_alert "Compressing" "${FINALDEST}/${version}.img.xz" "info" + pixz -9 < $DESTIMG/${version}.img > ${FINALDEST}/${version}.img.xz compression_type=".xz" fi if [[ $COMPRESS_OUTPUTIMAGE == *img* || $COMPRESS_OUTPUTIMAGE == *7z* ]]; then - mv $DESTIMG/${version}.img $DEST/images/${version}.img || exit 1 + mv $DESTIMG/${version}.img ${FINALDEST}/${version}.img || exit 1 compression_type="" fi if [[ $COMPRESS_OUTPUTIMAGE == *sha* ]]; then - cd $DEST/images + cd ${FINALDEST} display_alert "SHA256 calculating" "${version}.img${compression_type}" "info" sha256sum -b ${version}.img${compression_type} > ${version}.img${compression_type}.sha fi if [[ $COMPRESS_OUTPUTIMAGE == *gpg* ]]; then - cd $DEST/images + cd ${FINALDEST} if [[ -n $GPG_PASS ]]; then display_alert "GPG signing" "${version}.img${compression_type}" "info" - echo $GPG_PASS | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes $DEST/images/${version}.img${compression_type} || exit 1 + echo $GPG_PASS | gpg --passphrase-fd 0 --armor --detach-sign --pinentry-mode loopback --batch --yes ${FINALDEST}/${version}.img${compression_type} || exit 1 else display_alert "GPG signing skipped - no GPG_PASS" "${version}.img" "wrn" fi fi - fingerprint_image "$DEST/images/${version}.img${compression_type}.txt" "${version}" + fingerprint_image "${FINALDEST}/${version}.img${compression_type}.txt" "${version}" if [[ $COMPRESS_OUTPUTIMAGE == *7z* ]]; then - display_alert "Compressing" "$DEST/images/${version}.7z" "info" + display_alert "Compressing" "${FINALDEST}/${version}.7z" "info" 7za a -t7z -bd -m0=lzma2 -mx=3 -mfb=64 -md=32m -ms=on \ - $DEST/images/${version}.7z ${version}.key ${version}.img* >/dev/null 2>&1 - find $DEST/images/ -type \ + ${FINALDEST}/${version}.7z ${version}.key ${version}.img* >/dev/null 2>&1 + find ${FINALDEST}/ -type \ f \( -name "${version}.img" -o -name "${version}.img.asc" -o -name "${version}.img.txt" -o -name "${version}.img.sha" \) -print0 \ | xargs -0 rm >/dev/null 2>&1 fi rm -rf $DESTIMG fi - display_alert "Done building" "$DEST/images/${version}.img" "info" - - if [[ $BUILD_ALL == yes ]]; then - install -d -o nobody -g nogroup -m 775 $DEST/images/${BOARD}/{archive,nightly} - if [[ "$BETA" == yes ]]; then - install ${INSTALL_PARA} $DEST/images/"${version}"* $DEST/images/"${BOARD}"/nightly - rm $DEST/images/"${version}"* - else - install ${INSTALL_PARA} $DEST/images/"${version}"* $DEST/images/"${BOARD}"/archive - rm $DEST/images/"${version}"* - fi - fi + display_alert "Done building" "${FINALDEST}/${version}.img" "info" # call custom post build hook - [[ $(type -t post_build_image) == function ]] && post_build_image "$DEST/images/${version}.img" + [[ $(type -t post_build_image) == function ]] && post_build_image "${FINALDEST}/${version}.img" # write image to SD card - if [[ $(lsblk "$CARD_DEVICE" 2>/dev/null) && -f $DEST/images/${version}.img ]]; then + if [[ $(lsblk "$CARD_DEVICE" 2>/dev/null) && -f ${FINALDEST}/${version}.img ]]; then # make sha256sum if it does not exists. we need it for comparisson - if [[ -f "$DEST/images/${version}".img.sha ]]; then - local ifsha=$(cat $DEST/images/${version}.img.sha | awk '{print $1}') + if [[ -f "${FINALDEST}/${version}".img.sha ]]; then + local ifsha=$(cat ${FINALDEST}/${version}.img.sha | awk '{print $1}') else - local ifsha=$(sha256sum -b "$DEST/images/${version}".img | awk '{print $1}') + local ifsha=$(sha256sum -b "${FINALDEST}/${version}".img | awk '{print $1}') fi display_alert "Writing image" "$CARD_DEVICE ${readsha}" "info" # write to SD card - pv -p -b -r -c -N "[ .... ] dd" $DEST/images/${version}.img | dd of=$CARD_DEVICE bs=1M iflag=fullblock oflag=direct status=none + pv -p -b -r -c -N "[ .... ] dd" ${FINALDEST}/${version}.img | dd of=$CARD_DEVICE bs=1M iflag=fullblock oflag=direct status=none # read and compare display_alert "Verifying. Please wait!" - local ofsha=$(dd if=$CARD_DEVICE count=$(du -b $DEST/images/${version}.img | cut -f1) status=none iflag=count_bytes oflag=direct | sha256sum | awk '{print $1}') + local ofsha=$(dd if=$CARD_DEVICE count=$(du -b ${FINALDEST}/${version}.img | cut -f1) status=none iflag=count_bytes oflag=direct | sha256sum | awk '{print $1}') if [[ $ifsha == $ofsha ]]; then display_alert "Writing verified" "${version}.img" "info" else diff --git a/lib/general.sh b/lib/general.sh index b48caa48d..8ec56198a 100644 --- a/lib/general.sh +++ b/lib/general.sh @@ -205,7 +205,7 @@ improved_git() retries=0 break fi - let count=$count+1 + let count=$count+1 sleep $delay done @@ -866,25 +866,23 @@ repo-manipulate() { # $2: Architecture # $3: Amount of packages to keep repo-remove-old-packages() { - local repo=$1 - local arch=$2 - local keep=$3 - - for pkg in $(aptly repo search -config="${SCRIPTPATH}config/${REPO_CONFIG}" "${repo}" "Architecture ($arch)" | grep -v "ERROR: no results" | sort -rV); do - local pkg_name - pkg_name=$(echo "${pkg}" | cut -d_ -f1) - if [ "$pkg_name" != "$cur_pkg" ]; then - local count=0 - local deleted="" - local cur_pkg="$pkg_name" - fi - test -n "$deleted" && continue - ((count+=1)) - if [[ $count -gt $keep ]]; then - pkg_version=$(echo "${pkg}" | cut -d_ -f2) - aptly repo remove -config="${SCRIPTPATH}config/${REPO_CONFIG}" "${repo}" "Name ($pkg_name), Version (<= $pkg_version)" - deleted='yes' - fi + local repo=$1 + local arch=$2 + local keep=$3 + for pkg in $(aptly repo search -config="${SCRIPTPATH}config/${REPO_CONFIG}" "${repo}" "Architecture ($arch)" | grep -v "ERROR: no results" | sort -t '.' -nk4); do + local pkg_name + count=0 + pkg_name=$(echo "${pkg}" | cut -d_ -f1) + for subpkg in $(aptly repo search -config="${SCRIPTPATH}config/${REPO_CONFIG}" "${repo}" "Name ($pkg_name)" | grep -v "ERROR: no results" | sort -rt '.' -nk4); do + ((count+=1)) +# echo $subpkg + if [[ $count -gt $keep ]]; then + #echo "rem" + pkg_version=$(echo "${subpkg}" | cut -d_ -f2) + #echo $pkg_version + aptly repo remove -config="${SCRIPTPATH}config/${REPO_CONFIG}" "${repo}" "Name ($pkg_name), Version (= $pkg_version)" + fi + done done } diff --git a/lib/image-helpers.sh b/lib/image-helpers.sh index 40ba4c567..bb9d4a47e 100644 --- a/lib/image-helpers.sh +++ b/lib/image-helpers.sh @@ -145,7 +145,7 @@ install_deb_chroot() [[ $NO_APT_CACHER != yes ]] && local apt_extra="-o Acquire::http::Proxy=\"http://${APT_PROXY_ADDR:-localhost:3142}\" -o Acquire::http::Proxy::localhost=\"DIRECT\"" # when building in bulk from remote, lets make sure we have up2date index [[ $BUILD_ALL == yes && ${variant} == remote ]] && chroot "${SDCARD}" /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get $apt_extra -yqq update" - chroot "${SDCARD}" /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get -yqq \ - $apt_extra --no-install-recommends install $name" >> "${DEST}"/debug/install.log 2>&1 + chroot "${SDCARD}" /bin/bash -c "DEBIAN_FRONTEND=noninteractive apt-get -yqq $apt_extra --no-install-recommends install $name" >> "${DEST}"/debug/install.log 2>&1 + [[ $? -ne 0 ]] && exit_with_error "Installation of $name failed" "@host" [[ ${variant} == remote && ${transfer} == yes ]] && rsync -rq "${SDCARD}"/var/cache/apt/archives/*.deb ${DEB_STORAGE}/ }