JDM170/build
1
0
Fork 0
mirror of https://github.com/armbian/build synced 2025-09-24 19:47:06 +07:00
Code Issues Packages Projects Releases Wiki Activity

armbian-next: Python tooling: use consolidated+hashed+cached pip base/pycache; don't pip during Dockerfile build, nor cli-requirements

Browse Source 
- consolidate at `prepare_python_and_pip()`
- sanity check for Python version 3.9+ regardless of HOSTRELEASE
- TODO: pip vs sudo/root: need pip 22.2+ to curb warning, not doing it
This commit is contained in:
Ricardo Pardini
2023-01-13 04:37:30 +01:00
parent 350cf62ee0
commit 7beeae6219
7 changed files with 108 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
lib/functions/cli/cli-jsoninfo.sh
View File

@@ -6,17 +6,12 @@ function cli_json_info_pre_run() {
function cli_json_info_run() {
display_alert "Generating JSON info" "for all boards; wait" "info"
# So call a Python launcher.
# @TODO: this works without ti right now, since all the python stuff works with no external packages
# - python debian packages hostdeps? (-dev, -pip, virtualenv, etc)
# - run the virtualenv (messy?)
declare python3_binary_path
prepare_python3_binary_for_python_tools
obtain_and_check_host_release_and_arch # sets HOSTRELEASE
prepare_python_and_pip # requires HOSTRELEASE
# The info extractor itself...
run_host_command_logged "${python3_binary_path}" "${SRC}"/lib/tools/info.py ">" "${SRC}/output/info.json"
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${SRC}"/lib/tools/info.py ">" "${SRC}/output/info.json"
# Also convert output to CSV for easy import into Google Sheets etc
run_host_command_logged "${python3_binary_path}" "${SRC}"/lib/tools/json2csv.py "<" "${SRC}/output/info.json" ">" "${SRC}/output/info.csv"
run_host_command_logged "${PYTHON3_VARS[@]}" "${PYTHON3_INFO[BIN]}" "${SRC}"/lib/tools/json2csv.py "<" "${SRC}/output/info.json" ">" "${SRC}/output/info.csv"
}

6
lib/functions/cli/cli-requirements.sh
View File

@@ -17,11 +17,11 @@ function cli_requirements_pre_run() {
function cli_requirements_run() {
initialize_extension_manager # initialize the extension manager.
declare -a -g host_dependencies=()
obtain_and_check_host_release_and_arch # Sets HOSTRELEASE & validates it for sanity; also HOSTARCH
host_release="${HOSTRELEASE}" host_arch="${HOSTARCH}" early_prepare_host_dependencies
LOG_SECTION="install_host_dependencies" do_with_logging install_host_dependencies "for requirements command"
LOG_SECTION="prepare_pip_packages_for_python_tools" do_with_logging prepare_pip_packages_for_python_tools
display_alert "Done with" "@host dependencies" "cachehit"
}

11
lib/functions/compilation/kernel-patching.sh
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env bash
function kernel_main_patching_python() {
prepare_pip_packages_for_python_tools
prepare_python_and_pip
# outer scope variables: kernel_drivers_patch_file kernel_drivers_patch_hash
@@ -11,8 +11,7 @@ function kernel_main_patching_python() {
# array with all parameters; will be auto-quoted by bash's @Q modifier below
declare -a params_quoted=(
"PYTHONUNBUFFERED=yes" # Python should not buffer output, so we can see it in real time.
"PYTHONPYCACHEPREFIX=${SRC}/cache/pycache" # Python should not use its own cache, but use our own.
"${PYTHON3_VARS[@]}" # Default vars, from prepare_python_and_pip
"LOG_DEBUG=${patch_debug}" # Logging level for python.
"SRC=${SRC}" # Armbian root
"OUTPUT=${temp_file_for_output}" # Output file for the python script.
@@ -44,11 +43,9 @@ function kernel_main_patching_python() {
"EXTRA_PATCH_HASHES_FIRST=${kernel_drivers_patch_hash}" # Is a space-separated list.
)
display_alert "Calling Python patching script" "for kernel" "info"
declare python3_binary_path
prepare_python3_binary_for_python_tools
# "raw_command" is only for logging purposes.
raw_command="[...shortened kernel...] ${python3_binary_path} ${SRC}/lib/tools/patching.py" \
run_host_command_logged env -i "${params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/patching.py"
raw_command="[...shortened kernel patching...] ${PYTHON3_INFO[BIN]} ${SRC}/lib/tools/patching.py" \
run_host_command_logged env -i "${params_quoted[@]@Q}" "${PYTHON3_INFO[BIN]}" "${SRC}/lib/tools/patching.py"
run_host_command_logged cat "${temp_file_for_output}"
# shellcheck disable=SC1090
source "${temp_file_for_output}" # SOURCE IT!

12
lib/functions/compilation/uboot-patching.sh
View File

@@ -1,13 +1,12 @@
function uboot_main_patching_python() {
prepare_pip_packages_for_python_tools
prepare_python_and_pip
# outer scope variable: uboot_work_dir
temp_file_for_output="$(mktemp)" # Get a temporary file for the output.
# array with all parameters; will be auto-quoted by bash's @Q modifier below
declare -a params_quoted=(
"PYTHONUNBUFFERED=yes" # Python should not buffer output, so we can see it in real time.
"PYTHONPYCACHEPREFIX=${SRC}/cache/pycache" # Python should not use its own cache, but use our own.
"${PYTHON3_VARS[@]}" # Default vars, from prepare_python_and_pip
"LOG_DEBUG=${SHOW_DEBUG}" # Logging level for python.
"SRC=${SRC}" # Armbian root
"OUTPUT=${temp_file_for_output}" # Output file for the python script.
@@ -30,11 +29,10 @@ function uboot_main_patching_python() {
"BRANCH_FOR_PATCHES=u-boot-${BRANCH}-${BOARD}" # When applying patches-to-git, use this branch.
)
display_alert "Calling Python patching script" "for u-boot target" "info"
declare python3_binary_path
prepare_python3_binary_for_python_tools
# "raw_command" is only for logging purposes.
raw_command="[...shortened u-boot...] ${python3_binary_path} ${SRC}/lib/tools/patching.py" \
run_host_command_logged env -i "${params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/patching.py"
raw_command="[...shortened u-boot patching...] ${PYTHON3_INFO[BIN]} ${SRC}/lib/tools/patching.py" \
run_host_command_logged env -i "${params_quoted[@]@Q}" "${PYTHON3_INFO[BIN]}" "${SRC}/lib/tools/patching.py"
run_host_command_logged cat "${temp_file_for_output}"
# shellcheck disable=SC1090
source "${temp_file_for_output}" # SOURCE IT!

15
lib/functions/configuration/aggregation.sh
View File

@@ -15,16 +15,14 @@ function aggregate_packages() {
}
function aggregate_all_packages_python() {
prepare_pip_packages_for_python_tools # although aggregation can run without any package, it does benefit
prepare_python_and_pip
# Get a temporary file for the output. This is not WORKDIR yet, since we're still in configuration phase.
temp_file_for_aggregation="$(mktemp)"
# array with all parameters; will be auto-quoted by bash's @Q modifier below
declare -a aggregation_params_quoted=(
"PYTHONUNBUFFERED=yes" # Python should not buffer output, so we can see it in real time.
"PYTHONPYCACHEPREFIX=${SRC}/cache/pycache" # Python should not use its own cache, but use our own.
"${PYTHON3_VARS[@]}" # Default vars, from prepare_python_and_pip
"LOG_DEBUG=${SHOW_DEBUG}" # Logging level for python.
"SRC=${SRC}"
"OUTPUT=${temp_file_for_aggregation}"
@@ -72,13 +70,12 @@ function aggregate_all_packages_python() {
"PACKAGE_LIST_BOARD_REMOVE=${PACKAGE_LIST_BOARD_REMOVE}"
"PACKAGE_LIST_FAMILY_REMOVE=${PACKAGE_LIST_FAMILY_REMOVE}"
)
declare python3_binary_path
prepare_python3_binary_for_python_tools
# "raw_command" is only for logging purposes.
raw_command="[...shortened...] ${python3_binary_path} ${SRC}/lib/tools/aggregation.py" \
run_host_command_logged env -i "${aggregation_params_quoted[@]@Q}" "${python3_binary_path}" "${SRC}/lib/tools/aggregation.py"
raw_command="[...shortened...] ${PYTHON3_INFO[BIN]} ${SRC}/lib/tools/aggregation.py" \
run_host_command_logged env -i "${aggregation_params_quoted[@]@Q}" "${PYTHON3_INFO[BIN]}" "${SRC}/lib/tools/aggregation.py"
#run_host_command_logged cat "${temp_file_for_aggregation}"
# shellcheck disable=SC1090
source "${temp_file_for_aggregation}" # SOURCE IT!
run_host_command_logged rm "${temp_file_for_aggregation}"
run_host_command_logged rm -f "${temp_file_for_aggregation}"
}

111
lib/functions/general/python-tools.sh
View File

@@ -1,29 +1,100 @@
# This whole thing is a big "I refuse to use venv in a simple bash script" delusion.
# If you know to tame it, teach me. I'd rather not know about PYTHONUSERBASE and such.
# --rpardini
function early_prepare_pip3_dependencies_for_python_tools() {
# This is like a stupid version of requirements.txt
declare -a -g python3_pip_dependencies=(
"unidiff==0.7.4" # for parsing unified diff
"GitPython==3.1.29" # for manipulating git repos
"GitPython==3.1.30" # for manipulating git repos
"unidecode==1.3.6" # for converting strings to ascii
"coloredlogs==15.0.1" # for colored logging
)
return 0
}
function prepare_pip_packages_for_python_tools() {
early_prepare_pip3_dependencies_for_python_tools
# call: prepare_python_and_pip # this defines global PYTHON3_INFO dict and PYTHON3_VARS array
function prepare_python_and_pip() {
# First determine with python3 to use; requires knowing the HOSTRELEASE.
[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"
declare -g PYTHON_TOOLS_PIP_PACKAGES_DONE="${PYTHON_TOOLS_PIP_PACKAGES_DONE:-no}"
if [[ "${PYTHON_TOOLS_PIP_PACKAGES_DONE}" == "yes" ]]; then
display_alert "Required Python packages" "already installed" "info"
# fake-memoize this, it's expensive and does not need to be done twice
declare -g _already_prepared_python_and_pip="${_already_prepared_python_and_pip:-no}"
if [[ "${_already_prepared_python_and_pip}" == "yes" ]]; then
display_alert "All Python preparation done before" "skipping python prep" "debug"
return 0
fi
# @TODO: virtualenv? system-wide for now
display_alert "Installing required Python packages" "via pip3" "info"
declare python3_binary_path
prepare_python3_binary_for_python_tools
declare python3_binary_path="/usr/bin/python3"
declare python3_pip_bin_path="/usr/bin/pip3" # from hostdeps package python3-pip
# Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9.
if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then
python3_binary_path="/usr/bin/python3.9"
display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn"
fi
run_host_command_logged "${python3_binary_path}" /usr/bin/pip3 install "${python3_pip_dependencies[@]}"
# Check that the actual python3 --version is 3.9 at least
declare python3_version
python3_version="$("${python3_binary_path}" --version 2>&1 | cut -d' ' -f2)"
display_alert "Python3 version" "${python3_version}" "info"
if ! linux-version compare "${python3_version}" ge "3.9"; then
exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
fi
# Check actual pip3 version
declare pip3_version
pip3_version="$("${python3_binary_path}" "${python3_pip_bin_path}" --version 2>&1 | cut -d' ' -f2)"
display_alert "pip3 version" "${pip3_version}" "info"
# Hash the contents of the dependencies array + the Python version + the release
declare python3_pip_dependencies_hash
early_prepare_pip3_dependencies_for_python_tools
python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${python3_pip_dependencies[*]}" | sha256sum | cut -d' ' -f1)"
declare python_pip_cache="${SRC}/cache/pip"
declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
declare python3_user_base="${SRC}/cache/pip/base"
declare python3_pycache="${SRC}/cache/pip/pycache"
# declare a readonly global dict with all needed info for executing stuff using this setup
declare -r -g -A PYTHON3_INFO=(
[BIN]="${python3_binary_path}"
[USERBASE]="${python3_user_base}"
[PYCACHEPREFIX]="${python3_pycache}"
[HASH]="${python3_pip_dependencies_hash}"
[DEPS]="${python3_pip_dependencies[*]}"
[VERSION]="${python3_version}"
[PIP_VERSION]="${pip3_version}"
)
# declare a readonly global array for ENV vars to invoke python3 with
declare -r -g -a PYTHON3_VARS=(
"PYTHONUSERBASE=${PYTHON3_INFO[USERBASE]}"
"PYTHONUNBUFFERED=yes"
"PYTHONPYCACHEPREFIX=${PYTHON3_INFO[PYCACHEPREFIX]}"
)
# If the hash file exists, we're done.
if [[ -f "${python_hash_file}" ]]; then
display_alert "Using cached pip packages for Python tools" "${python3_pip_dependencies_hash}" "cachehit"
else
display_alert "Installing pip packages for Python tools" "${python3_pip_dependencies_hash:0:10}" "info"
# remove the old hashes matching base, don't leave junk behind
run_host_command_logged rm -fv "${python_hash_base}*"
# @TODO: when running with sudo:
# WARNING: The directory '/home/human/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.
# --root-user-action=ignore requires pip 22.1+
run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" "${python3_pip_bin_path}" install \
--no-warn-script-location --user "${python3_pip_dependencies[@]}"
# Create the hash file
run_host_command_logged touch "${python_hash_file}"
fi
_already_prepared_python_and_pip="yes"
return 0
}
@@ -35,23 +106,9 @@ function host_deps_add_extra_python() {
# host_release is from outer scope (
# Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9.
if [[ "focal ulyana ulyssa uma una" == *"${host_release}"* ]]; then
display_alert "Using Python 3.9 for" "'${host_release}' has outdated python3, using python3.9" "warn"
display_alert "Using Python 3.9 for" "hostdeps: '${host_release}' has outdated python3, using python3.9" "warn"
host_dependencies+=("python3.9-dev")
else
display_alert "Using Python3 for" "'${host_release}' has python3 >= 3.9" "debug"
fi
}
# This sets the outer scope variable 'python3_binary_path' to /usr/bin/python3 or similar, depending on version.
function prepare_python3_binary_for_python_tools() {
[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"
python3_binary_path="/usr/bin/python3"
# Determine what version of python3; focal-like OS's have Python 3.8, but we need 3.9.
if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then
python3_binary_path="/usr/bin/python3.9"
display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn"
else
display_alert "Using '${python3_binary_path}' for" "'$HOSTRELEASE' has python3 >= 3.9" "debug"
display_alert "Using Python3 for" "hostdeps: '${host_release}' has python3 >= 3.9" "debug"
fi
}

8
lib/functions/host/docker.sh
View File

@@ -115,11 +115,9 @@ function docker_cli_prepare() {
enable_all_extensions_builtin_and_user
initialize_extension_manager
fi
declare -a -g host_dependencies=() python3_pip_dependencies=()
early_prepare_pip3_dependencies_for_python_tools
declare -a -g host_dependencies=()
host_release="${wanted_release_tag}" early_prepare_host_dependencies
display_alert "Pre-game host dependencies" "${host_dependencies[*]}" "debug"
display_alert "Pre-game pip3 dependencies" "${python3_pip_dependencies[*]}" "debug"
#############################################################################################################
# Stop here if Docker can't be used at all.
@@ -129,7 +127,7 @@ function docker_cli_prepare() {
fi
#############################################################################################################
# Detect some docker info.
# Detect some docker info. @TODO: invoke "docker info" only once. really. it's very slow
DOCKER_SERVER_VERSION="$(docker info | grep -i -e "Server Version\:" | cut -d ":" -f 2 | xargs echo -n)"
display_alert "Docker Server version" "${DOCKER_SERVER_VERSION}" "debug"
@@ -222,8 +220,6 @@ function docker_cli_prepare() {
${c} RUN echo "--> CACHE MISS IN DOCKERFILE: apt packages." && \
${c} DEBIAN_FRONTEND=noninteractive apt-get -y update && \
${c} DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ${BASIC_DEPS[@]} ${host_dependencies[@]}
${c} RUN echo "--> CACHE MISS IN DOCKERFILE: pip3 packages." && \
${c} pip3 install ${python3_pip_dependencies[@]}
${c} RUN sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
${c} RUN locale-gen
WORKDIR ${DOCKER_ARMBIAN_TARGET_PATH}