mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build
- artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
This commit is contained in:
Ricardo Pardini
@@ -1,25 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This is already run under logging, don't use do_with_logging under here.
|
||||
function build_rootfs_only() {
|
||||
# validate that tmpfs_estimated_size is set and higher than zero, or exit_with_error
|
||||
[[ -z ${tmpfs_estimated_size} ]] && exit_with_error "tmpfs_estimated_size is not set"
|
||||
[[ ${tmpfs_estimated_size} -le 0 ]] && exit_with_error "tmpfs_estimated_size is not higher than zero"
|
||||
|
||||
# stage: prepare basic rootfs: unpack cache or create from scratch
|
||||
get_or_create_rootfs_cache_chroot_sdcard # only occurrence of this
|
||||
|
||||
# obtain the size, in MiB, of "${SDCARD}" at this point.
|
||||
declare -i rootfs_size_mib
|
||||
rootfs_size_mib=$(du -sm "${SDCARD}" | awk '{print $1}')
|
||||
display_alert "Actual rootfs size" "${rootfs_size_mib}MiB after basic/cache" ""
|
||||
|
||||
# warn if rootfs_size_mib is higher than the tmpfs_estimated_size
|
||||
if [[ ${rootfs_size_mib} -gt ${tmpfs_estimated_size} ]]; then
|
||||
display_alert "Rootfs actual size is larger than estimated tmpfs size after basic/cache" "${rootfs_size_mib}MiB > ${tmpfs_estimated_size}MiB" "wrn"
|
||||
fi
|
||||
}
|
||||
|
||||
# called by artifact-rootfs::artifact_rootfs_prepare_version()
|
||||
function calculate_rootfs_cache_id() {
|
||||
# Validate that AGGREGATED_ROOTFS_HASH is set
|
||||
[[ -z "${AGGREGATED_ROOTFS_HASH}" ]] && exit_with_error "AGGREGATED_ROOTFS_HASH is not set at calculate_rootfs_cache_id()"
|
||||
@@ -52,170 +33,82 @@ function calculate_rootfs_cache_id() {
|
||||
display_alert "calculate_rootfs_cache_id: done." "rootfs_cache_id: '${rootfs_cache_id}'" "debug"
|
||||
}
|
||||
|
||||
# this gets from cache or produces a basic new rootfs, ready, but not mounted, at "$SDCARD"
|
||||
function get_or_create_rootfs_cache_chroot_sdcard() {
|
||||
if [[ "${ROOT_FS_CREATE_ONLY}" == yes ]]; then
|
||||
exit_with_error "Using deprecated ROOT_FS_CREATE_ONLY=yes, that is not longer supported. use 'rootfs' CLI command."
|
||||
fi
|
||||
|
||||
# validate "${SDCARD}" is set. it does not exist, yet...
|
||||
if [[ -z "${SDCARD}" ]]; then
|
||||
exit_with_error "SDCARD is not set at get_or_create_rootfs_cache_chroot_sdcard()"
|
||||
fi
|
||||
[[ ! -d "${SDCARD:?}" ]] && exit_with_error "create_new_rootfs_cache: ${SDCARD} is not a directory"
|
||||
|
||||
# this was moved from configuration to this stage, that way configuration can be offline
|
||||
# if ROOTFSCACHE_VERSION not provided, check which is current version in the cache storage in GitHub.
|
||||
# - ROOTFSCACHE_VERSION is provided by external "build rootfs GHA script" in armbian/scripts
|
||||
if [[ -z "${ROOTFSCACHE_VERSION}" ]]; then
|
||||
if [[ "${SKIP_ARMBIAN_REPO}" != "yes" ]]; then
|
||||
display_alert "ROOTFSCACHE_VERSION not set, getting remotely" "Github API and armbian/mirror " "debug"
|
||||
# rpardini: why 2 calls?
|
||||
# this uses `jq` hostdep
|
||||
ROOTFSCACHE_VERSION=$(curl https://api.github.com/repos/armbian/cache/releases/latest -s --fail | jq .tag_name -r || true)
|
||||
# anonymous API access is very limited which is why we need a fallback
|
||||
# rpardini: yeah but this is 404'ing
|
||||
#ROOTFSCACHE_VERSION=${ROOTFSCACHE_VERSION:-$(curl -L --silent https://cache.armbian.com/rootfs/latest --fail)}
|
||||
display_alert "Remotely-obtained ROOTFSCACHE_VERSION" "${ROOTFSCACHE_VERSION}" "debug"
|
||||
else
|
||||
ROOTFSCACHE_VERSION=668 # The neighbour of the beast.
|
||||
display_alert "Armbian mirror skipped, using fictional rootfs cache version" "${ROOTFSCACHE_VERSION}" "debug"
|
||||
fi
|
||||
else
|
||||
display_alert "ROOTFSCACHE_VERSION is set externally" "${ROOTFSCACHE_VERSION}" "warn"
|
||||
fi
|
||||
|
||||
# Make ROOTFSCACHE_VERSION global at this point, in case it was not.
|
||||
declare -g ROOTFSCACHE_VERSION="${ROOTFSCACHE_VERSION}"
|
||||
|
||||
display_alert "ROOTFSCACHE_VERSION found online or preset" "${ROOTFSCACHE_VERSION}" "debug"
|
||||
|
||||
calculate_rootfs_cache_id # this sets packages_hash and cache_type
|
||||
|
||||
# seek last cache, proceed to previous otherwise build it
|
||||
local -a cache_list=()
|
||||
get_rootfs_cache_list_into_array_variable # sets cache_list
|
||||
|
||||
# Show the number of items in the cache_list array
|
||||
display_alert "Found possible rootfs caches: " "${#cache_list[@]}" "debug"
|
||||
|
||||
display_alert "ROOTFSCACHE_VERSION after getting cache list" "${ROOTFSCACHE_VERSION}" "debug"
|
||||
|
||||
declare possible_cached_version
|
||||
for possible_cached_version in "${cache_list[@]}"; do
|
||||
ROOTFSCACHE_VERSION="${possible_cached_version}" # global var
|
||||
local cache_name="${ARCH}-${RELEASE}-${cache_type}-${packages_hash}-${ROOTFSCACHE_VERSION}.tar.zst"
|
||||
local cache_fname="${SRC}/cache/rootfs/${cache_name}"
|
||||
|
||||
display_alert "Checking cache" "$cache_name" "info"
|
||||
|
||||
# if aria2 file exists download didn't succeeded
|
||||
if [[ ! -f $cache_fname || -f ${cache_fname}.aria2 ]]; then
|
||||
if [[ "${SKIP_ARMBIAN_REPO}" != "yes" ]]; then
|
||||
display_alert "Downloading from servers" # download_rootfs_cache() requires ROOTFSCACHE_VERSION
|
||||
download_and_verify "rootfs" "$cache_name" || continue
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -f $cache_fname && ! -f ${cache_fname}.aria2 ]]; then
|
||||
display_alert "Cache found!" "$cache_name" "info"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
display_alert "ROOTFSCACHE_VERSION after looping" "${ROOTFSCACHE_VERSION}" "debug"
|
||||
|
||||
# if cache found, extract it
|
||||
# if aria2 file exists, download didn't succeeded, so skip it
|
||||
# we can ignore existing cache with IGNORE_EXISTING_ROOTFS_CACHE=yes
|
||||
if [[ "${IGNORE_EXISTING_ROOTFS_CACHE}" != "yes" && -f "${cache_fname}" && ! -f "${cache_fname}.aria2" ]]; then
|
||||
# validate sanity
|
||||
[[ "x${SDCARD}x" == "xx" ]] && exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: SDCARD: ${SDCARD} is not set"
|
||||
|
||||
local date_diff=$((($(date +%s) - $(stat -c %Y "${cache_fname}")) / 86400))
|
||||
display_alert "Extracting $cache_name" "$date_diff days old" "info"
|
||||
pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "extract_rootfs") $cache_name" "$cache_fname" | zstdmt -dc | tar xp --xattrs -C "${SDCARD}"/
|
||||
|
||||
declare -a pv_tar_zstdmt_pipe_status=("${PIPESTATUS[@]}") # capture and the pipe_status array from PIPESTATUS
|
||||
declare one_pipe_status
|
||||
for one_pipe_status in "${pv_tar_zstdmt_pipe_status[@]}"; do
|
||||
if [[ "$one_pipe_status" != "0" ]]; then
|
||||
exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: ${cache_fname} failed (${pv_tar_zstdmt_pipe_status[*]}) - corrupt cache?"
|
||||
fi
|
||||
done
|
||||
|
||||
wait_for_disk_sync "after restoring rootfs cache"
|
||||
|
||||
run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf
|
||||
run_host_command_logged echo "nameserver ${NAMESERVER}" ">" "${SDCARD}"/etc/resolv.conf
|
||||
|
||||
create_sources_list "${RELEASE}" "${SDCARD}/"
|
||||
else
|
||||
display_alert "Creating rootfs" "cache miss" "info"
|
||||
create_new_rootfs_cache
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# called by artifact-rootfs::artifact_rootfs_build_from_sources()
|
||||
function create_new_rootfs_cache() {
|
||||
: "${artifact_final_file:?artifact_final_file is not set}"
|
||||
: "${artifact_final_file_basename:?artifact_final_file_basename is not set}"
|
||||
|
||||
[[ ! -d "${SDCARD:?}" ]] && exit_with_error "create_new_rootfs_cache: ${SDCARD} is not a directory"
|
||||
# validate cache_type is set
|
||||
[[ -n "${cache_type}" ]] || exit_with_error "create_new_rootfs_cache: cache_type is not set"
|
||||
# validate packages_hash is set
|
||||
[[ -n "${packages_hash}" ]] || exit_with_error "create_new_rootfs_cache: packages_hash is not set"
|
||||
|
||||
# This var ROOT_FS_CREATE_VERSION is only used here, afterwards it's all cache_name and cache_fname
|
||||
declare ROOT_FS_CREATE_VERSION="${ROOT_FS_CREATE_VERSION:-"$(date --utc +"%Y%m%d")"}"
|
||||
declare cache_name=${ARCH}-${RELEASE}-${cache_type}-${packages_hash}-${ROOT_FS_CREATE_VERSION}.tar.zst
|
||||
declare cache_fname=${SRC}/cache/rootfs/${cache_name}
|
||||
# compatibility with legacy code...
|
||||
declare -g cache_name="${artifact_final_file_basename}"
|
||||
declare -g cache_fname=${artifact_final_file}
|
||||
|
||||
display_alert "Creating new rootfs cache for" "'${RELEASE}' '${ARCH}' '${ROOT_FS_CREATE_VERSION}'" "info"
|
||||
display_alert "Creating new rootfs cache" "'${cache_name}'" "info"
|
||||
|
||||
create_new_rootfs_cache_via_debootstrap # in rootfs-create.sh
|
||||
create_new_rootfs_cache_tarball # in rootfs-create.sh
|
||||
|
||||
# needed for backend to keep current only @TODO: still needed?
|
||||
echo "$cache_fname" > "${cache_fname}.current"
|
||||
|
||||
# define a readonly global with the name of the cache
|
||||
declare -g -r BUILT_ROOTFS_CACHE_NAME="${cache_name}"
|
||||
declare -g -r BUILT_ROOTFS_CACHE_FILE="${cache_fname}"
|
||||
|
||||
return 0 # protect against possible future short-circuiting above this
|
||||
}
|
||||
|
||||
# return a list of versions of all available cache from remote and local into outer scoe "cache_list" variable
|
||||
function get_rootfs_cache_list_into_array_variable() {
|
||||
# If global vars are empty, exit_with_error
|
||||
[[ "x${ARCH}x" == "xx" ]] && exit_with_error "ARCH is not set"
|
||||
[[ "x${RELEASE}x" == "xx" ]] && exit_with_error "RELEASE is not set"
|
||||
[[ "x${packages_hash}x" == "xx" ]] && exit_with_error "packages_hash is not set"
|
||||
[[ "x${cache_type}x" == "xx" ]] && exit_with_error "cache_type is not set"
|
||||
# this builds/gets cached rootfs artifact, extracts it to "${SDCARD}"
|
||||
function get_or_create_rootfs_cache_chroot_sdcard() {
|
||||
if [[ "${ROOT_FS_CREATE_ONLY}" == yes ]]; then
|
||||
exit_with_error "Using deprecated ROOT_FS_CREATE_ONLY=yes, that is not longer supported. use 'rootfs' CLI command."
|
||||
fi
|
||||
|
||||
# this uses `jq` hostdep
|
||||
# build the rootfs artifact; capture the filename...
|
||||
declare -g artifact_final_file artifact_version artifact_final_file artifact_file_relative
|
||||
WHAT="rootfs" build_artifact_for_image # has its own logging sections, for now
|
||||
declare -g cache_fname="${artifact_final_file}"
|
||||
|
||||
declare -a local_cache_list=() # outer scope variable
|
||||
readarray -t local_cache_list <<< "$({
|
||||
# Don't even try remote if we're told to skip.
|
||||
if [[ "${SKIP_ARMBIAN_REPO}" != "yes" ]]; then
|
||||
curl --silent --fail -L "https://api.github.com/repos/armbian/cache/releases?per_page=3" | jq -r '.[].tag_name' ||
|
||||
curl --silent --fail -L https://cache.armbian.com/rootfs/list
|
||||
# Setup the cleanup handler, possibly "again", since the artifact already set it up and consumed it, if cache missed.
|
||||
LOG_SECTION="prepare_rootfs_build_params_and_trap" do_with_logging prepare_rootfs_build_params_and_trap
|
||||
|
||||
LOG_SECTION="extract_rootfs_artifact" do_with_logging extract_rootfs_artifact
|
||||
return 0
|
||||
}
|
||||
|
||||
function extract_rootfs_artifact() {
|
||||
: "${artifact_file_relative:?artifact_file_relative is not set}"
|
||||
: "${artifact_final_file:?artifact_final_file is not set}"
|
||||
# compatibility with legacy code...
|
||||
declare cache_name="${artifact_file_relative}"
|
||||
declare cache_fname=${artifact_final_file}
|
||||
|
||||
if [[ ! -f "${cache_fname}" ]]; then
|
||||
exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: ${cache_fname} is not a file"
|
||||
fi
|
||||
|
||||
# validate sanity
|
||||
[[ "x${SDCARD}x" == "xx" ]] && exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: SDCARD: ${SDCARD} is not set"
|
||||
[[ ! -d "${SDCARD}" ]] && exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: ${SDCARD} is not a directory"
|
||||
|
||||
# @TODO: validate SDCARD is empty; if not, the artifact build "leaked" a cleanup
|
||||
|
||||
local date_diff=$((($(date +%s) - $(stat -c %Y "${cache_fname}")) / 86400))
|
||||
display_alert "Extracting ${artifact_version}" "${date_diff} days old" "info"
|
||||
pv -p -b -r -c -N "$(logging_echo_prefix_for_pv "extract_rootfs") ${artifact_version}" "${cache_fname}" | zstdmt -dc | tar xp --xattrs -C "${SDCARD}"/
|
||||
|
||||
declare -a pv_tar_zstdmt_pipe_status=("${PIPESTATUS[@]}") # capture and the pipe_status array from PIPESTATUS
|
||||
declare one_pipe_status
|
||||
for one_pipe_status in "${pv_tar_zstdmt_pipe_status[@]}"; do
|
||||
if [[ "$one_pipe_status" != "0" ]]; then
|
||||
exit_with_error "get_or_create_rootfs_cache_chroot_sdcard: extract: ${cache_fname} failed (${pv_tar_zstdmt_pipe_status[*]}) - corrupt cache?"
|
||||
fi
|
||||
|
||||
find "${SRC}"/cache/rootfs/ -mtime -7 -name "${ARCH}-${RELEASE}-${cache_type}-${packages_hash}-*.tar.zst" |
|
||||
sed -e 's#^.*/##' |
|
||||
sed -e 's#\..*$##' |
|
||||
awk -F'-' '{print $5}'
|
||||
} | sort | uniq | sort -r)"
|
||||
|
||||
# Show the contents
|
||||
display_alert "Available cache versions number" "${#local_cache_list[*]}" "debug"
|
||||
# Loop each and show
|
||||
for cache_version in "${local_cache_list[@]}"; do
|
||||
display_alert "One available cache version" "${cache_version}" "debug"
|
||||
done
|
||||
|
||||
# return the list to outer scope
|
||||
cache_list=("${local_cache_list[@]}")
|
||||
wait_for_disk_sync "after restoring rootfs cache"
|
||||
|
||||
run_host_command_logged rm -v "${SDCARD}"/etc/resolv.conf
|
||||
run_host_command_logged echo "nameserver ${NAMESERVER}" ">" "${SDCARD}"/etc/resolv.conf
|
||||
|
||||
create_sources_list "${RELEASE}" "${SDCARD}/"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user