JDM170/build
1
0
Fork 0
mirror of https://github.com/armbian/build synced 2025-09-24 19:47:06 +07:00
Code Issues Packages Projects Releases Wiki Activity

armbian-next: remove warm/cold bundles from fetch_from_repo, replace with 2 invocation-specific hooks and gitballs (.tar of .git)

Browse Source 
- drop experiment 'caches/gitbundles' and its code; all lives now in separate repo
- change LINUXSOURCEDIR **again**, cleanup the old one
This commit is contained in:
Ricardo Pardini
2022-06-22 23:57:26 +02:00
parent 0aa57b5ef5
commit 95e0b9d484
4 changed files with 79 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
lib/functions/compilation/kernel.sh
View File

@@ -95,26 +95,67 @@ function compile_kernel() {
return 0
}
function kernel_prepare_git() {
if [[ -n $KERNELSOURCE ]]; then
[[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree"
function kernel_init_git_from_http_gitball() {
local kernel_git_dir="${1}"
display_alert "Downloading sources" "kernel" "git"
# Does not work well with rpi for example: GIT_WARM_REMOTE_SHALLOW_AT_TAG="v${KERNEL_MAJOR_MINOR}" \
# GIT_WARM_REMOTE_SHALLOW_AT_TAG sets GIT_WARM_REMOTE_SHALLOW_AT_DATE for you, as long as it is included by GIT_WARM_REMOTE_FETCH_TAGS
# GIT_WARM_REMOTE_SHALLOW_AT_DATE is the only one really used for making shallow
GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \
GIT_WARM_REMOTE_NAME="kernel-stable-${KERNEL_MAJOR_MINOR}" \
GIT_WARM_REMOTE_URL="${MAINLINE_KERNEL_SOURCE}" \
GIT_WARM_REMOTE_BRANCH="linux-${KERNEL_MAJOR_MINOR}.y" \
GIT_WARM_REMOTE_FETCH_TAGS="v${KERNEL_MAJOR_MINOR}*" \
GIT_WARM_REMOTE_SHALLOW_AT_TAG="${KERNEL_MAJOR_SHALLOW_TAG}" \
GIT_WARM_REMOTE_BUNDLE="kernel-stable-${KERNEL_MAJOR_MINOR}" \
GIT_COLD_BUNDLE_URL="${MAINLINE_KERNEL_COLD_BUNDLE_URL}" \
fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes"
local gitball_dir="${SRC}/cache/gitballs/kernel"
if [[ ! -d "${gitball_dir}" ]]; then
display_alert "Creating kernel git gitball cache dir" "${gitball_dir}" "info"
run_host_command_logged mkdir -pv "${gitball_dir}"
[[ -d "${SRC}/cache/gitbundles" ]] && run_host_command_logged rm -rf "${SRC}/cache/gitbundles" # remove old gitbundles dir. we won't be using those anymore; @TODO: remove this line in the future
fi
local gitball_file="${gitball_dir}/linux-${KERNEL_MAJOR_MINOR}.git.tar"
local gitball_url="https://github.com/armbian/gitutils/releases/download/latest/linux-${KERNEL_MAJOR_MINOR}.git.tar"
if [[ ! -f "${gitball_file}" ]]; then # Download the gitball file if it does not exist.
display_alert "Downloading Git cold gitball via HTTP" "${gitball_url}" "info" # This gonna take a while. And waste bandwidth
run_host_command_logged wget --continue --progress=dot:giga --output-document="${gitball_file}.tmp" "${gitball_url}"
run_host_command_logged mv -v "${gitball_file}.tmp" "${gitball_file}"
else
display_alert "Cold gitball file exists, using it" "${gitball_file}" "git"
fi
# Extract the gitball file to the git directory. This will create '.git'
run_host_command_logged tar -xvf "${gitball_file}" -C "${kernel_git_dir}"
# Sanity check
regular_git branch -a --list --color
}
function kernel_prepare_git_pre_fetch() {
local remote_name="kernel-stable-${KERNEL_MAJOR_MINOR}"
local remote_url="${MAINLINE_KERNEL_SOURCE}"
local remote_tags_to_fetch="v${KERNEL_MAJOR_MINOR}*"
# shellcheck disable=SC2154 # do_add_origin is defined in fetch_from_repo, and this is hook for it, so it's in context.
if [[ "${do_add_origin}" == "yes" ]]; then
display_alert "Fetching mainline stable tags" "${remote_name} tags: ${remote_tags_to_fetch}" "git"
regular_git remote add "${remote_name}" "${remote_url}" # Add the remote to the warmup source
# Fetch the tags. This allows working -rcX versions of still-unreleased minor versions.
improved_git_fetch "${remote_name}" "'refs/tags/${remote_tags_to_fetch}:refs/tags/${remote_tags_to_fetch}'" || true # Fetch the remote branch tags
display_alert "After mainline stable tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull
fi
}
function kernel_prepare_git() {
[[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then?
# LINUXSOURCEDIR has changed a lot, cleanup old incarnations if they exist
if [[ -d "${SRC}/cache/sources/kernel" ]]; then
display_alert "Cleaning up old kernel sources directory" "might take a while" "debug"
run_host_command_logged rm -rf "${SRC}/cache/sources/kernel"
fi
[[ -d "${kernel_work_dir}" ]] && cd "${kernel_work_dir}" && fasthash_debug "pre git, existing tree"
display_alert "Downloading sources" "kernel" "git"
GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \
GIT_INIT_REPO_HOOK=kernel_init_git_from_http_gitball \
GIT_PRE_FETCH_HOOK=kernel_prepare_git_pre_fetch_tags \
fetch_from_repo "$KERNELSOURCE" "unused:set via GIT_FIXED_WORKDIR" "$KERNELBRANCH" "yes"
}
function kernel_maybe_clean() {

2
lib/functions/configuration/main-config.sh
View File

@@ -120,7 +120,7 @@ function do_main_configuration() {
MAINLINE_FIRMWARE_SOURCE='https://mirrors.bfsu.edu.cn/git/linux-firmware.git'
;;
*)
MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git'
MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git' # "linux-stable" was renamed to "linux"
MAINLINE_FIRMWARE_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git'
;;
esac

196
lib/functions/general/git.sh
View File

@@ -109,17 +109,23 @@ fetch_from_repo() {
fi
fi
local do_warmup_remote="no" do_cold_bundle="no" do_add_origin="no"
local do_add_origin="no"
if [[ "$(git rev-parse --git-dir)" != ".git" ]]; then
# Dir is not a git working copy. Make it so.
display_alert "Creating local copy" "$dir $ref_name"
regular_git init -q --initial-branch="armbian_unused_initial_branch" .
# Dir is not a git working copy. Make it so;
# If callback is defined, call it. Give it the dir as param. The rest it will read from environment.
# If not callback defined, do an init, and schedule a fetch.
if [[ $(type -t ${GIT_INIT_REPO_HOOK} || true) == function ]]; then
display_alert "Delegating to ${GIT_INIT_REPO_HOOK}()" "git init: $dir $ref_name" "debug"
${GIT_INIT_REPO_HOOK} "${git_work_dir}"
else
display_alert "Initializing empty git local copy" "git init: $dir $ref_name"
regular_git init -q --initial-branch="armbian_unused_initial_branch" .
fi
offline=false # Force online, we'll need to fetch.
do_add_origin="yes" # Just created the repo, it needs an origin later.
do_warmup_remote="yes" # Just created the repo, mark it as ready to receive the warm remote if exists.
do_cold_bundle="yes" # Just created the repo, mark it as ready to receive a cold bundle if that is available.
# @TODO: possibly hang a cleanup handler here: if this fails, ${git_work_dir} should be removed.
fi
local changed=false
@@ -163,7 +169,11 @@ fetch_from_repo() {
local checkout_from="HEAD" # Probably best to use the local revision?
if [[ "${changed}" == "true" ]]; then
git_handle_cold_and_warm_bundle_remotes # Delegate to function to find or create cache if appropriate.
if [[ $(type -t ${GIT_PRE_FETCH_HOOK} || true) == function ]]; then
display_alert "Delegating to ${GIT_PRE_FETCH_HOOK}()" "before git fetch" "debug"
${GIT_PRE_FETCH_HOOK} "${git_work_dir}" "${url}" "$ref_type" "$ref_name"
fi
if [[ "${do_add_origin}" == "yes" ]]; then
regular_git remote add origin "${url}"
@@ -193,11 +203,6 @@ fetch_from_repo() {
#fasthash_debug "before git clean of $dir $ref_name"
regular_git clean -q -d -f # Files that are not tracked by git and were added when the patch was applied must be removed.
# set the checkout date on all the versioned files.
# @TODO: this is contentious. disable for now. patches will still use the mininum date set by checked_out_revision_mtime above
#git ls-tree -r -z --name-only "${checkout_from}" | xargs -0 -- touch -m -t "${checked_out_revision_mtime:0:12}.${checked_out_revision_mtime:12}"
#fasthash_debug "after setting checkout time for $dir $ref_name" #yeah
if [[ -f .gitmodules ]]; then
if [[ "${GIT_SKIP_SUBMODULES}" == "yes" ]]; then
display_alert "Skipping submodules" "GIT_SKIP_SUBMODULES=yes" "debug"
@@ -224,168 +229,3 @@ fetch_from_repo() {
#fasthash_debug "at the end of fetch_from_repo $dir $ref_name"
}
function git_fetch_from_bundle_file() {
local bundle_file="${1}" remote_name="${2}" shallow_file="${3}"
regular_git bundle verify "${bundle_file}" # Make sure bundle is valid.
regular_git remote add "${remote_name}" "${bundle_file}" # Add the remote pointing to the cold bundle file
if [[ -f "${shallow_file}" ]]; then
display_alert "Bundle is shallow" "${shallow_file}" "git"
cp -p "${shallow_file}" ".git/shallow"
fi
improved_git_fetch --tags "${remote_name}" # Fetch it! (including tags!)
display_alert "Bundle fetch '${remote_name}' completed, working copy size" "$(du -h -s | awk '{print $1}')" "git"
}
function download_git_bundle_from_http() {
local bundle_file="${1}" bundle_url="${2}"
if [[ ! -f "${git_cold_bundle_cache_file}" ]]; then # Download the bundle file if it does not exist.
display_alert "Downloading Git cold bundle via HTTP" "${bundle_url}" "info" # This gonna take a while. And waste bandwidth
run_host_command_logged wget --continue --progress=dot:giga --output-document="${bundle_file}" "${bundle_url}"
else
display_alert "Cold bundle file exists, using it" "${bundle_file}" "git"
fi
}
function git_remove_cold_and_warm_bundle_remotes() {
# Remove the cold bundle remote, otherwise it holds references that impede the shallow to actually work.
if [[ ${has_git_cold_remote} -gt 0 ]]; then
regular_git remote remove "${git_cold_bundle_remote_name}"
has_git_cold_remote=0
fi
# Remove the warmup remote, otherwise it holds references forever.
if [[ ${has_git_warm_remote} -gt 0 ]]; then
regular_git remote remove "${GIT_WARM_REMOTE_NAME}"
has_git_warm_remote=0
fi
}
function git_handle_cold_and_warm_bundle_remotes() {
local has_git_cold_remote=0
local has_git_warm_remote=0
local git_warm_remote_bundle_file git_warm_remote_bundle_cache_dir git_warm_remote_bundle_file_shallowfile
local git_warm_remote_bundle_extra_fn=""
# First check the warm remote bundle cache. If that exists, use that, and skip the cold bundle.
if [[ "${do_warmup_remote}" == "yes" ]]; then
if [[ "${GIT_WARM_REMOTE_NAME}" != "" ]] && [[ "${GIT_WARM_REMOTE_BUNDLE}" != "" ]]; then
# Add extras to filename, for shallow by tag or revision
if [[ "${GIT_WARM_REMOTE_SHALLOW_REVISION}" != "" ]]; then
git_warm_remote_bundle_extra_fn="-shallow-rev-${GIT_WARM_REMOTE_SHALLOW_REVISION}"
elif [[ "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" != "" ]]; then
git_warm_remote_bundle_extra_fn="-shallow-tag-${GIT_WARM_REMOTE_SHALLOW_AT_TAG}"
fi
git_warm_remote_bundle_cache_dir="${SRC}/cache/gitbundles/warm" # calculate the id, dir and name of local file and remote
git_warm_remote_bundle_file="${git_warm_remote_bundle_cache_dir}/${GIT_WARM_REMOTE_BUNDLE}${git_warm_remote_bundle_extra_fn}.gitbundle" # final filename of bundle
git_warm_remote_bundle_file_shallowfile="${git_warm_remote_bundle_file}.shallow" # it can be there's a shallow revision
if [[ -f "${git_warm_remote_bundle_file}" ]]; then
display_alert "Fetching from warm git bundle, wait" "${GIT_WARM_REMOTE_BUNDLE}" "info" # This is gonna take a long while...
git_fetch_from_bundle_file "${git_warm_remote_bundle_file}" "${GIT_WARM_REMOTE_NAME}" "${git_warm_remote_bundle_file_shallowfile}"
do_cold_bundle="no" # Skip the cold bundle, below.
do_warmup_remote="no" # Skip the warm bundle creation, below, too.
has_git_warm_remote=1 # mark warm remote as added.
else
display_alert "Could not find warm bundle file" "${git_warm_remote_bundle_file}" "git"
fi
fi
fi
if [[ "${do_cold_bundle}" == "yes" ]]; then
# If there's a cold bundle URL specified:
# - if there's already a cold_bundle_xxx remote, move on.
# - grab the bundle via http/https first, and fetch from that, into "cold_bundle_xxx" remote.
# - do nothing else with this, it'll be used internally by git to avoid a huge fetch later.
# - but, after this, the wanted branch will be fetched. signal has_git_cold_remote=1 for later.
if [[ "${GIT_COLD_BUNDLE_URL}" != "" ]]; then
local git_cold_bundle_id git_cold_bundle_cache_dir git_cold_bundle_cache_file git_cold_bundle_remote_name
git_cold_bundle_cache_dir="${SRC}/cache/gitbundles/cold" # calculate the id, dir and name of local file and remote
git_cold_bundle_id="$(echo -n "${GIT_COLD_BUNDLE_URL}" | md5sum | awk '{print $1}')" # md5 of the URL.
git_cold_bundle_cache_file="${git_cold_bundle_cache_dir}/${git_cold_bundle_id}.gitbundle" # final filename of bundle
git_cold_bundle_remote_name="cold_bundle_${git_cold_bundle_id}" # name of the remote that will point to bundle
mkdir -p "${git_cold_bundle_cache_dir}" # make sure directory exists before downloading
download_git_bundle_from_http "${git_cold_bundle_cache_file}" "${GIT_COLD_BUNDLE_URL}"
display_alert "Fetching from cold git bundle, wait" "${git_cold_bundle_id}" "info" # This is gonna take a long while...
git_fetch_from_bundle_file "${git_cold_bundle_cache_file}" "${git_cold_bundle_remote_name}"
has_git_cold_remote=1 # marker for pruning logic below
fi
fi
# If there's a warmup remote specified.
# - if there's a cached warmup bundle file, add it as remote and fetch from it, and move on.
# - add the warmup as remote, fetch from it; export it as a cached bundle for next time.
if [[ "${do_warmup_remote}" == "yes" ]]; then
if [[ "${GIT_WARM_REMOTE_NAME}" != "" ]] && [[ "${GIT_WARM_REMOTE_URL}" != "" ]] && [[ "${GIT_WARM_REMOTE_BRANCH}" != "" ]]; then
display_alert "Using Warmup Remote before origin fetch" "${GIT_WARM_REMOTE_NAME} - ${GIT_WARM_REMOTE_BRANCH}" "git"
regular_git remote add "${GIT_WARM_REMOTE_NAME}" "${GIT_WARM_REMOTE_URL}" # Add the remote to the warmup source
has_git_warm_remote=1 # mark as done. Will export the bundle!
improved_git_fetch --no-tags "${GIT_WARM_REMOTE_NAME}" "${GIT_WARM_REMOTE_BRANCH}" # Fetch the remote branch, but no tags
display_alert "After warm bundle, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull
# Checkout that to a branch. We wanna have a local reference to what has been fetched.
# @TODO: could be a param instead of FETCH_HEAD; would drop commits after that rev
local git_warm_branch_name="warm__${GIT_WARM_REMOTE_BRANCH}"
regular_git branch "${git_warm_branch_name}" FETCH_HEAD || true
improved_git_fetch "${GIT_WARM_REMOTE_NAME}" "'refs/tags/${GIT_WARM_REMOTE_FETCH_TAGS}:refs/tags/${GIT_WARM_REMOTE_FETCH_TAGS}'" || true # Fetch the remote branch, but no tags
display_alert "After warm bundle tags, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after bundle pull
# Lookup the tag (at the warm remote directly) to find the rev to shallow to.
if [[ "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" != "" ]]; then
display_alert "GIT_WARM_REMOTE_SHALLOW_AT_TAG" "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}" "git"
GIT_WARM_REMOTE_SHALLOW_AT_DATE="$(git tag --list --format="%(creatordate)" "${GIT_WARM_REMOTE_SHALLOW_AT_TAG}")"
display_alert "GIT_WARM_REMOTE_SHALLOW_AT_TAG ${GIT_WARM_REMOTE_SHALLOW_AT_TAG} resulted in GIT_WARM_REMOTE_SHALLOW_AT_DATE" "Date: ${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" "git"
fi
# At this stage, we might wanna make the local copy shallow and re-pack it.
if [[ "${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" != "" ]]; then
display_alert "Making working copy shallow" "before date ${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" "info"
# 'git clone' is the only consistent, usable thing we can do to do this.
# it does require a temporary dir, though. use one.
local temp_git_dir="${git_work_dir}.making.shallow.temp"
rm -rf "${temp_git_dir}"
regular_git clone --no-checkout --progress --verbose \
--single-branch --branch="${git_warm_branch_name}" \
--tags --shallow-since="${GIT_WARM_REMOTE_SHALLOW_AT_DATE}" \
"file://${git_work_dir}" "${temp_git_dir}"
display_alert "After shallow clone, temp_git_dir" "$(du -h -s "${temp_git_dir}" | awk '{print $1}')" "git" # Show size after shallow
# Get rid of original, replace with new. Move cwd so no warnings are produced.
cd "${SRC}" || exit_with_error "Failed to move cwd away so we can remove" "${git_work_dir}"
rm -rf "${git_work_dir}"
mv -v "${temp_git_dir}" "${git_work_dir}"
cd "${git_work_dir}" || exit_with_error "Failed to get new dir after clone" "${git_work_dir}"
# dir switched, no more the original remotes. but origin is leftover, remove it
regular_git remote remove origin || true
has_git_cold_remote=0
has_git_warm_remote=0
display_alert "After shallow, working copy size" "$(du -h -s | awk '{print $1}')" "git" # Show size after shallow
fi
# Now git working copy has a precious state we might wanna preserve (export the bundle).
if [[ "${GIT_WARM_REMOTE_BUNDLE}" != "" ]]; then
mkdir -p "${git_warm_remote_bundle_cache_dir}"
display_alert "Exporting warm remote bundle" "${git_warm_remote_bundle_file}" "info"
regular_git bundle create "${git_warm_remote_bundle_file}" --all
rm -f "${git_warm_remote_bundle_file_shallowfile}" # not shallow at first...
if [[ -f ".git/shallow" ]]; then
display_alert "Exported bundle is shallow" "Will copy to ${git_warm_remote_bundle_file_shallowfile}" "git"
cp -p ".git/shallow" "${git_warm_remote_bundle_file_shallowfile}"
fi
fi
fi
fi
# Make sure to remove the cold and warm bundle remote, otherwise it holds references for no good reason.
git_remove_cold_and_warm_bundle_remotes
}

2
lib/functions/main/config-prepare.sh
View File

@@ -155,7 +155,7 @@ function prepare_and_config_main_build_single() {
exit_with_error "Kernel series unsupported" "'${KERNEL_MAJOR_MINOR}' is unsupported, or bad config"
fi
export LINUXSOURCEDIR="kernel/${ARCH}__${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}"
export LINUXSOURCEDIR="linux-kernel/${KERNEL_MAJOR_MINOR}__${LINUXFAMILY}/${ARCH}"
else
export KERNEL_HAS_WORKING_HEADERS="yes" # I assume non-Armbian kernels have working headers, eg: Debian/Ubuntu generic do.
export ARMBIAN_WILL_BUILD_KERNEL=no