mirror of
https://github.com/armbian/build
synced 2025-09-24 19:47:06 +07:00
- also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between882f995e21and31ac6383e1armbian-next: manual merge (30) of family/board changes between882f995e21and31ac6383e1armbian-next: manual merge (29) of family/board changes between3435c46367and882f995e21(A LOT!) armbian-next: manual merge (29) of lib changes between3435c46367and882f995e21(A LOT!) armbian-next: manual merge (28) of lib changes between revisionsaf6ceee6c5and38df56fbf3armbian-next: manual merge (28) of sources/families changes between revisionsaf6ceee6c5and38df56fbf3armbian-next: manual merge (27) of `lib` changes between revisions9c52562176andaf6ceee6c5armbian-next: manual merge (27) of `sources/families` changes between revisions9c52562176andaf6ceee6c5armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions20ee8c5450and9c52562176armbian-next: manual merge (26) of lib changes between revisions20ee8c5450and9c52562176- @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisionsfe972621c6and20ee8c5450- @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisionsfe972621c6and20ee8c5450armbian-next: manual merge (24) of families changes between revisions9ca9120420and560531a635armbian-next: manual merge (24) of lib changes between revisions9ca9120420and560531a635armbian-next: manual merge (23) of all changes between revisions17b4fb913cand9ca9120420armbian-next: manual merge (22) of all changes between revisions0eb8fe7497and1dddf78cd0- @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisionse7d7dab1bband0eb8fe7497armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions6b72ae3c86and247c4c45fdarmbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisionsb23498b949ande621d25adc- the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts. - why? this proves we got a reproducible kernel modules build! - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions08cf31de73andc8855aa08d- heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform() - only runs during build, for now (not on device) - build_custom_uboot() - allow fully custom, extension driven, building of u-boot - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions64410fb74band08cf31de73- changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisionsbe9b5156a4and2a8e1ecac1- many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions0f7200c793and101eaec907armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions13469fd8a9and09e416e31c- also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions34d4be6b7band5fe0f36aa8armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revisioned589b248a- this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revisioned589b248aarmbian-next: manual merge (11) of all lib/*.sh changes between revisions3305d45b81anded589b248aarmbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info - include logs for failed configs too - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later - keeping the arm64 makefile Image.gz vs Image hack - fix order of packaging patch byteshift, but still there - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisionsa4ae3a2270and3305d45b81- but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revisionff4346c468armbian-next: manual merge (9) of all lib/*.sh changes between revisions3b7f5b1f34andff4346c468armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions1d499d9ac2and3b7f5b1f34armbian-next: manual merge (7) of all lib/*.sh changes between revisionsd885bfc97dand1d499d9ac2armbian-next: manual merge (6) of all lib/*.sh changes between revisionsc7f3c239feandd885bfc97darmbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file. - refactor that into a function as to be easy to if-out-of - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes` - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes` - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes` - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions1b18df3c83ande7962bb2b5- most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration() - but, they're just _set_ there, dirs are not created early, but on demand later - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still) - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly. - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now. - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors. - mostly FIP tooling invocations - but also the boot_logo image builder - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions23afccf56eande610f00bc7- plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions1035905760ande4e4ab0791- missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions117633687eand3083038855Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisionsf6143eff67andf3388b9aeeSigned-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli - shellfmt: auto-downloader and runner of shellfmt - darwin/linux - amd64/arm64 - find ~correct files to format - run formatting - check formatting soon - refactor compile's CLI stuff out into function/cli - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else - plus rebasing fixes - logging: refactorings, pt. 2: error handling - fix: no errors during umount_chroot() - no progress for CI=true builds - WiP disable kernel hashing. too crazy. - a few builds now working with "set -e" - wtf. or something - kernel logging and long_running stuff - a mess - needs moving around in files - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command - better logging for u-boot - more fixes, u-boot - more fixes for logging et al - git stuff - many (many) fixes - new color scheme - a monster. make sure unmounted at the end. remove set -e's, to-be-readded. - remove set -e "for safety reasons" - more alerts. we gotta know whats failing - some more logging stuff and fixes for error checking - more logging and error handling stuff - fixes; some set -e's - more logging stuff - logging: refactoring codebase, pt.1: functions-only - Refactor the hell out of everything into functions - rename build-all-ng to build-multi; other fixes, extensions init - slight nudge - some were already good, like this one. - syntax fixes - some need a little nudge - another clean one - some just need a better name (and splitting later) - syntax fixes - some were already good, like this desktop one - some were already good, like this other one - some were already good, like this one. - debootstrap is gone. - extract functions from compile.sh - add logging to main_default_build - more stuff - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
427 lines
24 KiB
Bash
427 lines
24 KiB
Bash
#!/usr/bin/env bash
|
|
# global variables managing the state of the extension manager. treat as private.
|
|
declare -A extension_function_info # maps a function name to a string with KEY=VALUEs information about the defining extension
|
|
declare -i initialize_extension_manager_counter=0 # how many times has the extension manager initialized?
|
|
declare -A defined_hook_point_functions # keeps a map of hook point functions that were defined and their extension info
|
|
declare -A hook_point_function_trace_sources # keeps a map of hook point functions that were actually called and their source
|
|
declare -A hook_point_function_trace_lines # keeps a map of hook point functions that were actually called and their source
|
|
declare fragment_manager_cleanup_file # this is a file used to cleanup the manager's produced functions, for build_all_ng
|
|
# configuration. Command line might override this.
|
|
export DEBUG_EXTENSION_CALLS=no # set to yes to log every hook function called to the main build log
|
|
export LOG_ENABLE_EXTENSION=yes # colorful logs with stacktrace when enable_extension is called.
|
|
|
|
# This is a helper function for calling hooks.
|
|
# It follows the pattern long used in the codebase for hook-like behaviour:
|
|
# [[ $(type -t name_of_hook_function) == function ]] && name_of_hook_function
|
|
# but with the following added behaviors:
|
|
# 1) it allows for many arguments, and will treat each as a hook point.
|
|
# this allows for easily kept backwards compatibility when renaming hooks, for example.
|
|
# 2) it will read the stdin and assume it's (Markdown) documentation for the hook point.
|
|
# combined with heredoc in the call site, it allows for "inline" documentation about the hook
|
|
# notice: this is not involved in how the hook functions came to be. read below for that.
|
|
call_extension_method() {
|
|
# First, consume the stdin and write metadata about the call.
|
|
write_hook_point_metadata "$@"
|
|
|
|
# @TODO: hack to handle stdin again, possibly with '< /dev/tty'
|
|
|
|
# Then a sanity check, hook points should only be invoked after the manager has initialized.
|
|
if [[ ${initialize_extension_manager_counter} -lt 1 ]]; then
|
|
display_alert "Extension problem" "Call to call_extension_method() (in ${BASH_SOURCE[1]- $(get_extension_hook_stracktrace "${BASH_SOURCE[*]}" "${BASH_LINENO[*]}")}) before extension manager is initialized." "err"
|
|
fi
|
|
|
|
# With DEBUG_EXTENSION_CALLS, log the hook call. Users might be wondering what/when is a good hook point to use, and this is visual aid.
|
|
[[ "${DEBUG_EXTENSION_CALLS}" == "yes" ]] &&
|
|
display_alert "--> Extension Method '${1}' being called from" "$(get_extension_hook_stracktrace "${BASH_SOURCE[*]}" "${BASH_LINENO[*]}")" ""
|
|
|
|
# Then call the hooks, if they are defined.
|
|
for hook_name in "$@"; do
|
|
echo "-- Extension Method being called: ${hook_name}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
if [[ $(type -t ${hook_name} || true) == function ]]; then
|
|
${hook_name}
|
|
fi
|
|
done
|
|
}
|
|
|
|
# what this does is a lot of bash mumbo-jumbo to find all board-,family-,config- or user-defined hook points.
|
|
# the meat of this is 'compgen -A function', which is bash builtin that lists all defined functions.
|
|
# it will then compose a full hook point (function) that calls all the implementing hooks.
|
|
# this centralized function will then be called by the regular Armbian build system, which is oblivious to how
|
|
# it came to be. (although it is encouraged to call hook points via call_extension_method() above)
|
|
# to avoid hard coding the list of hook-points (eg: user_config, image_tweaks_pre_customize, etc) we use
|
|
# a marker in the function names, namely "__" (two underscores) to determine the hook point.
|
|
initialize_extension_manager() {
|
|
# before starting, auto-add extensions specified (eg, on the command-line) via the ENABLE_EXTENSIONS env var. Do it only once.
|
|
[[ ${initialize_extension_manager_counter} -lt 1 ]] && [[ "${ENABLE_EXTENSIONS}" != "" ]] && {
|
|
local auto_extension
|
|
for auto_extension in $(echo "${ENABLE_EXTENSIONS}" | tr "," " "); do
|
|
ENABLE_EXTENSION_TRACE_HINT="ENABLE_EXTENSIONS -> " enable_extension "${auto_extension}"
|
|
done
|
|
}
|
|
|
|
# This marks the manager as initialized, no more extensions are allowed to load after this.
|
|
export initialize_extension_manager_counter=$((initialize_extension_manager_counter + 1))
|
|
|
|
# Extensions has its own work/tmp directory, defined by do_main_configuration, with build UUID. We just create it here, unless told not to.
|
|
display_alert "Initializing EXTENSION_MANAGER_TMP_DIR" "${EXTENSION_MANAGER_TMP_DIR}" "debug"
|
|
mkdir -p "${EXTENSION_MANAGER_TMP_DIR}"
|
|
|
|
# Log destination.
|
|
export EXTENSION_MANAGER_LOG_FILE="${LOGDIR}/999.extensions.log"
|
|
[[ "${WRITE_EXTENSIONS_METADATA:-yes}" == "no" ]] && echo -n "" > "${EXTENSION_MANAGER_TMP_DIR}/hook_point_calls.txt"
|
|
|
|
# Add trap handler to cleanup and not leave garbage behind when exiting.
|
|
add_cleanup_handler cleanup_handler_extensions
|
|
|
|
# globally initialize the extensions log.
|
|
echo "-- lib/extensions.sh included. logs will be below, followed by the debug generated by the initialize_extension_manager() function." > "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# log whats happening.
|
|
echo "-- initialize_extension_manager() called." >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# this is the all-important separator.
|
|
local hook_extension_delimiter="__"
|
|
|
|
# list all defined functions. filter only the ones that have the delimiter. get only the part before the delimiter.
|
|
# sort them, and make them unique. the sorting is required for uniq to work, and does not affect the ordering of execution.
|
|
# get them on a single line, space separated.
|
|
local all_hook_points
|
|
all_hook_points="$(compgen -A function | grep "${hook_extension_delimiter}" | awk -F "${hook_extension_delimiter}" '{print $1}' | sort | uniq | xargs echo -n)"
|
|
|
|
declare -i hook_points_counter=0 hook_functions_counter=0 hook_point_functions_counter=0
|
|
|
|
# initialize the cleanups file.
|
|
fragment_manager_cleanup_file="${EXTENSION_MANAGER_TMP_DIR}/extension_function_cleanup.sh"
|
|
echo "# cleanups: " > "${fragment_manager_cleanup_file}"
|
|
|
|
local FUNCTION_SORT_OPTIONS="--general-numeric-sort --ignore-case" # --random-sort could be used to introduce chaos
|
|
local hook_point=""
|
|
# now loop over the hook_points.
|
|
for hook_point in ${all_hook_points}; do
|
|
echo "-- hook_point ${hook_point}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# check if the hook point is already defined as a function.
|
|
# that can happen for example with user_config(), that can be implemented itself directly by a userpatches config.
|
|
# for now, just warn, but we could devise a way to actually integrate it in the call list.
|
|
# or: advise the user to rename their user_config() function to something like user_config__make_it_awesome()
|
|
local existing_hook_point_function
|
|
existing_hook_point_function="$(compgen -A function | grep "^${hook_point}\$" || true)"
|
|
if [[ "${existing_hook_point_function}" == "${hook_point}" ]]; then
|
|
echo "--- hook_point_functions (final sorted realnames): ${hook_point_functions}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
display_alert "Extension conflict" "function ${hook_point} already defined! ignoring functions: $(compgen -A function | grep "^${hook_point}${hook_extension_delimiter}")" "wrn"
|
|
continue
|
|
fi
|
|
|
|
# for each hook_point, obtain the list of implementing functions.
|
|
# the sort order here is (very) relevant, since it determines final execution order.
|
|
# so the name of the functions actually determine the ordering.
|
|
local hook_point_functions hook_point_functions_pre_sort hook_point_functions_sorted_by_sort_id
|
|
|
|
# Sorting. Multiple extensions (or even the same extension twice) can implement the same hook point
|
|
# as long as they have different function names (the part after the double underscore __).
|
|
# the order those will be called depends on the name; eg:
|
|
# 'hook_point__033_be_awesome()' would be caller sooner than 'hook_point__799_be_even_more_awesome()'
|
|
# independent from where they were defined or in which order the extensions containing them were added.
|
|
# since requiring specific ordering could hamper portability, we reward extension authors who
|
|
# don't mind ordering for writing just: 'hook_point__be_just_awesome()' which is automatically rewritten
|
|
# as 'hook_point__500_be_just_awesome()'.
|
|
# extension authors who care about ordering can use the 3-digit number, and use the context variables
|
|
# HOOK_ORDER and HOOK_POINT_TOTAL_FUNCS to confirm in which order they're being run.
|
|
|
|
# gather the real names of the functions (after the delimiter).
|
|
hook_point_functions_pre_sort="$(compgen -A function | grep "^${hook_point}${hook_extension_delimiter}" | awk -F "${hook_extension_delimiter}" '{print $2}' | xargs echo -n)"
|
|
echo "--- hook_point_functions_pre_sort: ${hook_point_functions_pre_sort}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# add "500_" to the names of function that do NOT start with a number.
|
|
# keep a reference from the new names to the old names (we'll sort on the new, but invoke the old)
|
|
declare -A hook_point_functions_sortname_to_realname
|
|
declare -A hook_point_functions_realname_to_sortname
|
|
for hook_point_function_realname in ${hook_point_functions_pre_sort}; do
|
|
local sort_id="${hook_point_function_realname}"
|
|
[[ ! $sort_id =~ ^[0-9] ]] && sort_id="500_${sort_id}"
|
|
hook_point_functions_sortname_to_realname[${sort_id}]="${hook_point_function_realname}"
|
|
hook_point_functions_realname_to_sortname[${hook_point_function_realname}]="${sort_id}"
|
|
done
|
|
|
|
# actually sort the sort_id's...
|
|
# shellcheck disable=SC2086
|
|
hook_point_functions_sorted_by_sort_id="$(echo "${hook_point_functions_realname_to_sortname[*]}" | tr " " "\n" | LC_ALL=C sort ${FUNCTION_SORT_OPTIONS} | xargs echo -n)"
|
|
echo "--- hook_point_functions_sorted_by_sort_id: ${hook_point_functions_sorted_by_sort_id}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# then map back to the real names, keeping the order..
|
|
hook_point_functions=""
|
|
for hook_point_function_sortname in ${hook_point_functions_sorted_by_sort_id}; do
|
|
hook_point_functions="${hook_point_functions} ${hook_point_functions_sortname_to_realname[${hook_point_function_sortname}]}"
|
|
done
|
|
# shellcheck disable=SC2086
|
|
hook_point_functions="$(echo -n ${hook_point_functions})"
|
|
echo "--- hook_point_functions (final sorted realnames): ${hook_point_functions}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
hook_point_functions_counter=0
|
|
hook_points_counter=$((hook_points_counter + 1))
|
|
|
|
# determine the variables we'll pass to the hook function during execution.
|
|
# this helps the extension author create extensions that are portable between userpatches and official Armbian.
|
|
# shellcheck disable=SC2089
|
|
local common_function_vars="HOOK_POINT=\"${hook_point}\""
|
|
|
|
# loop over the functions for this hook_point (keep a total for the hook point and a grand running total)
|
|
for hook_point_function in ${hook_point_functions}; do
|
|
hook_point_functions_counter=$((hook_point_functions_counter + 1))
|
|
hook_functions_counter=$((hook_functions_counter + 1))
|
|
done
|
|
common_function_vars="${common_function_vars} HOOK_POINT_TOTAL_FUNCS=\"${hook_point_functions_counter}\""
|
|
|
|
echo "-- hook_point: ${hook_point} will run ${hook_point_functions_counter} functions: ${hook_point_functions}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
local temp_source_file_for_hook_point="${EXTENSION_MANAGER_TMP_DIR}/extension_function_definition.sh"
|
|
|
|
hook_point_functions_loop_counter=0
|
|
|
|
# prepare the cleanup for the function, so we can remove our mess at the end of the build.
|
|
cat <<- FUNCTION_CLEANUP_FOR_HOOK_POINT >> "${fragment_manager_cleanup_file}"
|
|
unset ${hook_point}
|
|
FUNCTION_CLEANUP_FOR_HOOK_POINT
|
|
|
|
# now compose a function definition. notice the heredoc. it will be written to tmp file, logged, then sourced.
|
|
# theres a lot of opportunities here, but for now I keep it simple:
|
|
# - execute functions in the order defined by ${hook_point_functions} above
|
|
# - define call-specific environment variables, to help extension authors to write portable extensions (eg: EXTENSION_DIR)
|
|
cat <<- FUNCTION_DEFINITION_HEADER > "${temp_source_file_for_hook_point}"
|
|
${hook_point}() {
|
|
echo "*** Extension-managed hook starting '${hook_point}': will run ${hook_point_functions_counter} functions: '${hook_point_functions}'" >>"\${EXTENSION_MANAGER_LOG_FILE}"
|
|
FUNCTION_DEFINITION_HEADER
|
|
|
|
for hook_point_function in ${hook_point_functions}; do
|
|
hook_point_functions_loop_counter=$((hook_point_functions_loop_counter + 1))
|
|
|
|
# store the full name in a hash, so we can track which were actually called later.
|
|
defined_hook_point_functions["${hook_point}${hook_extension_delimiter}${hook_point_function}"]="DEFINED=yes ${extension_function_info["${hook_point}${hook_extension_delimiter}${hook_point_function}"]}"
|
|
|
|
# prepare the call context
|
|
local hook_point_function_variables="${common_function_vars}" # start with common vars... (eg: HOOK_POINT_TOTAL_FUNCS)
|
|
# add the contextual extension info for the function (eg, EXTENSION_DIR)
|
|
hook_point_function_variables="${hook_point_function_variables} ${extension_function_info["${hook_point}${hook_extension_delimiter}${hook_point_function}"]}"
|
|
# add the current execution counter, so the extension author can know in which order it is being actually called
|
|
hook_point_function_variables="${hook_point_function_variables} HOOK_ORDER=\"${hook_point_functions_loop_counter}\""
|
|
|
|
# add it to our (not the call site!) environment. if we export those in the call site, the stack is corrupted.
|
|
eval "${hook_point_function_variables}"
|
|
|
|
# output the call, passing arguments, and also logging the output to the extensions log.
|
|
# attention: don't pipe here (eg, capture output), otherwise hook function cant modify the environment (which is mostly the point)
|
|
cat <<- FUNCTION_DEFINITION_CALLSITE >> "${temp_source_file_for_hook_point}"
|
|
hook_point_function_trace_sources["${hook_point}${hook_extension_delimiter}${hook_point_function}"]="\${BASH_SOURCE[*]}"
|
|
hook_point_function_trace_lines["${hook_point}${hook_extension_delimiter}${hook_point_function}"]="\${BASH_LINENO[*]}"
|
|
[[ "\${DEBUG_EXTENSION_CALLS}" == "yes" ]] && display_alert "---> Extension Method ${hook_point}" "${hook_point_functions_loop_counter}/${hook_point_functions_counter} (ext:${EXTENSION:-built-in}) ${hook_point_function}" ""
|
|
echo "*** *** Extension-managed hook starting ${hook_point_functions_loop_counter}/${hook_point_functions_counter} '${hook_point}${hook_extension_delimiter}${hook_point_function}':" >>"\${EXTENSION_MANAGER_LOG_FILE}"
|
|
${hook_point_function_variables} ${hook_point}${hook_extension_delimiter}${hook_point_function} "\$@"
|
|
echo "*** *** Extension-managed hook finished ${hook_point_functions_loop_counter}/${hook_point_functions_counter} '${hook_point}${hook_extension_delimiter}${hook_point_function}':" >>"\${EXTENSION_MANAGER_LOG_FILE}"
|
|
FUNCTION_DEFINITION_CALLSITE
|
|
|
|
# output the cleanup for the implementation as well.
|
|
cat <<- FUNCTION_CLEANUP_FOR_HOOK_POINT_IMPLEMENTATION >> "${fragment_manager_cleanup_file}"
|
|
unset ${hook_point}${hook_extension_delimiter}${hook_point_function}
|
|
FUNCTION_CLEANUP_FOR_HOOK_POINT_IMPLEMENTATION
|
|
|
|
# unset extension vars for the next loop.
|
|
unset EXTENSION EXTENSION_DIR EXTENSION_FILE EXTENSION_ADDED_BY
|
|
done
|
|
|
|
cat <<- FUNCTION_DEFINITION_FOOTER >> "${temp_source_file_for_hook_point}"
|
|
echo "*** Extension-managed hook ending '${hook_point}': completed." >>"\${EXTENSION_MANAGER_LOG_FILE}"
|
|
} # end ${hook_point}() function
|
|
FUNCTION_DEFINITION_FOOTER
|
|
|
|
# unsets, lest the next loop inherits them
|
|
unset hook_point_functions hook_point_functions_sortname_to_realname hook_point_functions_realname_to_sortname
|
|
|
|
# log what was produced in our own debug logfile
|
|
cat "${temp_source_file_for_hook_point}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
cat "${fragment_manager_cleanup_file}" >> "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
# source the generated function.
|
|
# shellcheck disable=SC1090
|
|
source "${temp_source_file_for_hook_point}"
|
|
|
|
rm -f "${temp_source_file_for_hook_point}"
|
|
done
|
|
|
|
# Dont show any output until we have more than 1 hook function (we implement one already, below)
|
|
[[ ${hook_functions_counter} -gt 0 ]] &&
|
|
display_alert "Extension manager" "processed ${hook_points_counter} Extension Methods calls and ${hook_functions_counter} Extension Method implementations" "info" | tee -a "${EXTENSION_MANAGER_LOG_FILE}"
|
|
|
|
return 0 # exit with success, short-circuit above.
|
|
}
|
|
|
|
cleanup_extension_manager() {
|
|
if [[ -f "${fragment_manager_cleanup_file}" ]]; then
|
|
display_alert "Cleaning up" "extension manager" "info"
|
|
# shellcheck disable=SC1090 # dynamic source, thanks, shellcheck
|
|
source "${fragment_manager_cleanup_file}" # this will unset all the functions.
|
|
rm -f "${fragment_manager_cleanup_file}" # also remove the file.
|
|
unset fragment_manager_cleanup_file # and unset the var.
|
|
fi
|
|
|
|
# reset/unset the variables used
|
|
initialize_extension_manager_counter=0
|
|
unset extension_function_info defined_hook_point_functions hook_point_function_trace_sources hook_point_function_trace_lines fragment_manager_cleanup_file
|
|
}
|
|
|
|
function cleanup_handler_extensions() {
|
|
display_alert "yeah the extensions trap handler..." "cleanup_handler_extensions" "cleanup"
|
|
cleanup_extension_manager
|
|
|
|
# Stop logging.
|
|
unset EXTENSION_MANAGER_LOG_FILE
|
|
|
|
# cleanup our tmpdir.
|
|
if [[ -d "${EXTENSION_MANAGER_TMP_DIR}" ]]; then
|
|
rm -rf "${EXTENSION_MANAGER_TMP_DIR}"
|
|
fi
|
|
unset EXTENSION_MANAGER_TMP_DIR
|
|
}
|
|
|
|
# why not eat our own dog food?
|
|
# process everything that happened during extension related activities
|
|
# and write it to the log. also, move the log from the .tmp dir to its
|
|
# final location. this will make run_after_build() "hot" (eg, emit warnings)
|
|
run_after_build__999_finish_extension_manager() {
|
|
# export these maps, so the hook can access them and produce useful stuff.
|
|
export defined_hook_point_functions hook_point_function_trace_sources
|
|
|
|
# eat our own dog food, pt2.
|
|
call_extension_method "extension_metadata_ready" <<- 'EXTENSION_METADATA_READY'
|
|
*meta-Meta time!*
|
|
Implement this hook to work with/on the meta-data made available by the extension manager.
|
|
Interesting stuff to process:
|
|
- `"${EXTENSION_MANAGER_TMP_DIR}/hook_point_calls.txt"` contains a list of all hook points called, in order.
|
|
- For each hook_point in the list, more files will have metadata about that hook point.
|
|
- `${EXTENSION_MANAGER_TMP_DIR}/hook_point.orig.md` contains the hook documentation at the call site (inline docs), hopefully in Markdown format.
|
|
- `${EXTENSION_MANAGER_TMP_DIR}/hook_point.compat` contains the compatibility names for the hooks.
|
|
- `${EXTENSION_MANAGER_TMP_DIR}/hook_point.exports` contains _exported_ environment variables.
|
|
- `${EXTENSION_MANAGER_TMP_DIR}/hook_point.vars` contains _all_ environment variables.
|
|
- `${defined_hook_point_functions}` is a map of _all_ the defined hook point functions and their extension information.
|
|
- `${hook_point_function_trace_sources}` is a map of all the hook point functions _that were really called during the build_ and their BASH_SOURCE information.
|
|
- `${hook_point_function_trace_lines}` is the same, but BASH_LINENO info.
|
|
After this hook is done, the `${EXTENSION_MANAGER_TMP_DIR}` will be removed.
|
|
EXTENSION_METADATA_READY
|
|
}
|
|
|
|
# This is called by call_extension_method(). To say the truth, this should be in an extension. But then it gets too meta for anyone's head.
|
|
write_hook_point_metadata() {
|
|
# Dont do anything if told not to.
|
|
[[ "${WRITE_EXTENSIONS_METADATA:-yes}" == "no" ]] && return 0
|
|
|
|
local main_hook_point_name="$1"
|
|
|
|
[[ ! -d "${EXTENSION_MANAGER_TMP_DIR}" ]] && mkdir -p "${EXTENSION_MANAGER_TMP_DIR}"
|
|
cat - > "${EXTENSION_MANAGER_TMP_DIR}/${main_hook_point_name}.orig.md" # Write the hook point documentation received via stdin to a tmp file for later processing.
|
|
shift
|
|
echo -n "$@" > "${EXTENSION_MANAGER_TMP_DIR}/${main_hook_point_name}.compat" # log the 2nd+ arguments too (those are the alternative/compatibility names), separate file.
|
|
compgen -A export > "${EXTENSION_MANAGER_TMP_DIR}/${main_hook_point_name}.exports" # capture the exported env vars.
|
|
compgen -A variable > "${EXTENSION_MANAGER_TMP_DIR}/${main_hook_point_name}.vars" # capture all env vars.
|
|
|
|
# add to the list of hook points called, in order.
|
|
echo "${main_hook_point_name}" >> "${EXTENSION_MANAGER_TMP_DIR}/hook_point_calls.txt"
|
|
}
|
|
|
|
# can be called by board, family, config or user to make sure an extension is included.
|
|
# single argument is the extension name.
|
|
# will look for it in /userpatches/extensions first.
|
|
# if not found there will look in /extensions
|
|
# if not found will exit 17
|
|
declare -i enable_extension_recurse_counter=0
|
|
declare -a enable_extension_recurse_stack
|
|
enable_extension() {
|
|
local extension_name="$1"
|
|
local extension_dir extension_file extension_file_in_dir extension_floating_file
|
|
local stacktrace
|
|
|
|
# if LOG_ENABLE_EXTENSION, output useful stack, so user can figure out which extensions are being added where
|
|
if [[ "${LOG_ENABLE_EXTENSION}" == "yes" ]]; then
|
|
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
|
|
stacktrace="${ENABLE_EXTENSION_TRACE_HINT}$(get_extension_hook_stracktrace "${BASH_SOURCE[*]}" "${BASH_LINENO[*]}")"
|
|
display_alert "Enabling extension" "${extension_name} :: added by ${stacktrace}" "debug"
|
|
else
|
|
display_alert "Enabling extension" "${extension_name}" ""
|
|
fi
|
|
fi
|
|
|
|
# first a check, has the extension manager already initialized? then it is too late to enable_extension(). bail.
|
|
if [[ ${initialize_extension_manager_counter} -gt 0 ]]; then
|
|
stacktrace="${ENABLE_EXTENSION_TRACE_HINT}$(get_extension_hook_stracktrace "${BASH_SOURCE[*]}" "${BASH_LINENO[*]}")"
|
|
display_alert "Extension problem" "already initialized -- too late to add '${extension_name}' (trace: ${stacktrace})" "err"
|
|
exit 2
|
|
fi
|
|
|
|
# check the counter. if recurring, add to the stack and return success
|
|
if [[ $enable_extension_recurse_counter -gt 1 ]]; then
|
|
enable_extension_recurse_stack+=("${extension_name}")
|
|
return 0
|
|
fi
|
|
|
|
# increment the counter
|
|
enable_extension_recurse_counter=$((enable_extension_recurse_counter + 1))
|
|
|
|
# there are many opportunities here. too many, actually. let userpatches override just some functions, etc.
|
|
for extension_base_path in "${SRC}/userpatches/extensions" "${SRC}/extensions"; do
|
|
extension_dir="${extension_base_path}/${extension_name}"
|
|
extension_file_in_dir="${extension_dir}/${extension_name}.sh"
|
|
extension_floating_file="${extension_base_path}/${extension_name}.sh"
|
|
|
|
if [[ -d "${extension_dir}" ]] && [[ -f "${extension_file_in_dir}" ]]; then
|
|
extension_file="${extension_file_in_dir}"
|
|
break
|
|
elif [[ -f "${extension_floating_file}" ]]; then
|
|
extension_dir="${extension_base_path}" # this is misleading. only directory-based extensions should have this.
|
|
extension_file="${extension_floating_file}"
|
|
break
|
|
fi
|
|
done
|
|
|
|
# After that, we should either have extension_file and extension_dir, or throw.
|
|
if [[ ! -f "${extension_file}" ]]; then
|
|
echo "ERR: Extension problem -- cant find extension '${extension_name}' anywhere - called by ${BASH_SOURCE[1]}"
|
|
exit 17 # exit, forcibly. no way we can recover from this, and next extensions will get bogus errors as well.
|
|
fi
|
|
|
|
local before_function_list after_function_list new_function_list
|
|
|
|
# store a list of existing functions at this point, before sourcing the extension.
|
|
before_function_list="$(compgen -A function)"
|
|
|
|
# shellcheck disable=SC1090
|
|
source "${extension_file}"
|
|
|
|
# decrement the recurse counter, so calls to this method are allowed again.
|
|
enable_extension_recurse_counter=$((enable_extension_recurse_counter - 1))
|
|
|
|
# get a new list of functions after sourcing the extension
|
|
after_function_list="$(compgen -A function)"
|
|
|
|
# compare before and after, thus getting the functions defined by the extension.
|
|
# comm is oldskool. we like it. go "man comm" to understand -13 below
|
|
new_function_list="$(comm -13 <(echo "$before_function_list" | sort) <(echo "$after_function_list" | sort))"
|
|
|
|
# iterate over defined functions, store them in global associative array extension_function_info
|
|
for newly_defined_function in ${new_function_list}; do
|
|
#echo "func: ${newly_defined_function} has DIR: ${extension_dir}"
|
|
extension_function_info["${newly_defined_function}"]="EXTENSION=\"${extension_name}\" EXTENSION_DIR=\"${extension_dir}\" EXTENSION_FILE=\"${extension_file}\" EXTENSION_ADDED_BY=\"${stacktrace}\""
|
|
done
|
|
|
|
# snapshot, then clear, the stack
|
|
local -a stack_snapshot=("${enable_extension_recurse_stack[@]}")
|
|
enable_extension_recurse_stack=()
|
|
|
|
# process the stacked snapshot, finally enabling the extensions
|
|
for stacked_extension in "${stack_snapshot[@]}"; do
|
|
ENABLE_EXTENSION_TRACE_HINT="RECURSE ${stacktrace} ->" enable_extension "${stacked_extension}"
|
|
done
|
|
|
|
}
|
|
|
|
# Fancy placeholder for future ideas. allow any core function to be hooked. maybe with "voters" infrastructure?
|
|
function do_with_hooks() {
|
|
"$@"
|
|
}
|