armbian-build/lib/functions/compilation/uboot.sh

563 lines
26 KiB
Bash
Raw Permalink Normal View History

#!/usr/bin/env bash
#
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
#
# This file is a part of the Armbian Build Framework
# https://github.com/armbian/build/
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
function maybe_make_clean_uboot() {
if [[ $CLEAN_LEVEL == *make-uboot* ]]; then
display_alert "${uboot_prefix}Cleaning u-boot tree - CLEAN_LEVEL contains 'make-uboot'" "${BOOTSOURCEDIR}" "info"
2022-10-08 11:17:30 +00:00
(
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
cd "${SRC}/cache/sources/${BOOTSOURCEDIR}" || exit_with_error "crazy about ${BOOTSOURCEDIR}"
run_host_command_logged make clean
2022-10-08 11:17:30 +00:00
)
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
else
display_alert "${uboot_prefix}Not cleaning u-boot tree, use CLEAN_LEVEL=make-uboot if needed" "CLEAN_LEVEL=${CLEAN_LEVEL}" "debug"
fi
}
function patch_uboot_target() {
local uboot_work_dir=""
uboot_work_dir="$(pwd)"
# outer scope variable: uboot_git_revision, validate that it is set
if [[ -z "${uboot_git_revision}" ]]; then
exit_with_error "uboot_git_revision is not set"
fi
display_alert "${uboot_prefix} Checking out to clean sources SHA1 ${uboot_git_revision}" "{$BOOTSOURCEDIR} for ${target_make}"
git checkout -f -q "${uboot_git_revision}"
# remove all git untracked files; echo their names to screen
# this throws away the baby with the bathwater; rebuilds will be slow. but the risk of shipping wrong binaries is too high.
display_alert "${uboot_prefix} Cleaning u-boot tree" "${BOOTSOURCEDIR} for '${target_make}'"
regular_git clean -xfdq
maybe_make_clean_uboot
# Python patching for u-boot!
do_with_hooks uboot_main_patching_python
# create patch for manual source changes
if [[ $CREATE_PATCHES == yes ]]; then
userpatch_create "u-boot"
fi
}
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# this receives version target uboot_name uboottempdir uboot_target_counter toolchain as variables.
# also receives uboot_prefix, target_make, target_patchdir, target_files as input
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
function compile_uboot_target() {
: "${artifact_version:?artifact_version is not set}"
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
display_alert "${uboot_prefix}Listing contents of u-boot directory" "'${version}' '${target_make}' before patching" "debug"
run_host_command_logged "ls -laht"
fi
patch_uboot_target
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
display_alert "${uboot_prefix}Listing contents of u-boot directory" "'${version}' '${target_make}' after patching" "debug"
run_host_command_logged "ls -laht"
fi
if [[ $CREATE_PATCHES == yes ]]; then
return 0
fi
# atftempdir comes from atf.sh's compile_atf()
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
if [[ -n $ATFSOURCE && -d "${atftempdir}" ]]; then
display_alert "Copying over bin/elf/itb's from atftempdir" "${atftempdir}" "debug"
run_host_command_logged cp -pv "${atftempdir}"/*.bin "${atftempdir}"/*.elf "${atftempdir}"/*.itb ./ # only works due to nullglob
# atftempdir is under WORKDIR, so no cleanup necessary.
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
fi
2023-06-14 11:54:07 +00:00
# crusttempdir comes from crust.sh's compile_crust()
if [[ -n $CRUSTSOURCE && -d "${crusttempdir}" ]]; then
display_alert "Copying over bin/elf's from crusttempdir" "${crusttempdir}" "debug"
run_host_command_logged cp -pv "${crusttempdir}"/*.bin "${crusttempdir}"/*.elf ./ # only works due to nullglob
fi
# Hook time, for extra post-processing
call_extension_method "pre_config_uboot_target" <<- 'PRE_CONFIG_UBOOT_TARGET'
*allow extensions prepare before configuring and compiling an u-boot target*
Some u-boot targets require extra configuration or pre-processing before compiling.
For example, changing Python version can be done by replacing the `${BIN_WORK_DIR}/python` symlink.
PRE_CONFIG_UBOOT_TARGET
display_alert "${uboot_prefix}Preparing u-boot config '${BOOTCONFIG}'" "${version} ${target_make}" "info"
declare -g if_error_detail_message="${uboot_prefix}Failed to configure u-boot ${version} $BOOTCONFIG ${target_make}"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
run_host_command_logged CCACHE_BASEDIR="$(pwd)" PATH="${toolchain}:${toolchain2}:${PATH}" \
"KCFLAGS=-fdiagnostics-color=always" \
pipetty make "${CTHREADS}" "${BOOTCONFIG}" "CROSS_COMPILE=\"${CCACHE} ${UBOOT_COMPILER}\""
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# armbian specifics u-boot settings
[[ -f .config ]] && sed -i "s/CONFIG_LOCALVERSION=\"\"/CONFIG_LOCALVERSION=\"-armbian-${artifact_version}\"/g" .config
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
[[ -f .config ]] && sed -i 's/CONFIG_LOCALVERSION_AUTO=.*/# CONFIG_LOCALVERSION_AUTO is not set/g' .config
# for modern (? 2018-2019?) kernel and non spi targets @TODO: this does not belong here
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
if [[ ${BOOTBRANCH} =~ ^tag:v201[8-9](.*) && ${target} != "spi" && -f .config ]]; then
display_alert "Hacking ENV stuff in u-boot config 2018-2019" "for ${target}" "debug"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
sed -i 's/^.*CONFIG_ENV_IS_IN_FAT.*/# CONFIG_ENV_IS_IN_FAT is not set/g' .config
sed -i 's/^.*CONFIG_ENV_IS_IN_EXT4.*/CONFIG_ENV_IS_IN_EXT4=y/g' .config
sed -i 's/^.*CONFIG_ENV_IS_IN_MMC.*/# CONFIG_ENV_IS_IN_MMC is not set/g' .config
sed -i 's/^.*CONFIG_ENV_IS_NOWHERE.*/# CONFIG_ENV_IS_NOWHERE is not set/g' .config
echo "# CONFIG_ENV_IS_NOWHERE is not set" >> .config
echo 'CONFIG_ENV_EXT4_INTERFACE="mmc"' >> .config
echo 'CONFIG_ENV_EXT4_DEVICE_AND_PART="0:auto"' >> .config
echo 'CONFIG_ENV_EXT4_FILE="/boot/boot.env"' >> .config
fi
# @TODO: this does not belong here
[[ -f tools/logos/udoo.bmp ]] && run_host_command_logged cp -pv "${SRC}"/packages/blobs/splash/udoo.bmp tools/logos/udoo.bmp
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# @TODO: why?
touch .scmversion
# use `scripts/config` instead of sed if available. Cleaner results.
if [[ ! -f scripts/config ]]; then
display_alert "scripts/config not found" "u-boot ${version} $BOOTCONFIG ${target_make}" "debug"
# Old, pre-2022.10 u-boot; does not have the scripts/config script. Do it the old way.
# $BOOTDELAY can be set in board family config, ensure autoboot can be stopped even if set to 0
[[ $BOOTDELAY == 0 ]] && echo -e "CONFIG_ZERO_BOOTDELAY_CHECK=y" >> .config
[[ -n $BOOTDELAY ]] && sed -i "s/^CONFIG_BOOTDELAY=.*/CONFIG_BOOTDELAY=${BOOTDELAY}/" .config || [[ -f .config ]] && echo "CONFIG_BOOTDELAY=${BOOTDELAY}" >> .config
else
display_alert "scripts/config found" "u-boot ${version} $BOOTCONFIG ${target_make}" "debug"
# $BOOTDELAY can be set in board family config, ensure autoboot can be stopped even if set to 0
if [[ $BOOTDELAY == 0 ]]; then
display_alert "Adding CONFIG_ZERO_BOOTDELAY_CHECK=y u-boot config" "BOOTDELAY==0 for ${target}" "info"
run_host_command_logged scripts/config --enable CONFIG_ZERO_BOOTDELAY_CHECK
fi
# If BOOTDELAY is set, either change a preexisting CONFIG_BOOTDELAY or add it
if [[ -n $BOOTDELAY ]]; then
display_alert "Hacking autoboot delay in u-boot config" "BOOTDELAY=${BOOTDELAY} for ${target}" "info"
run_host_command_logged scripts/config --set-val CONFIG_BOOTDELAY "${BOOTDELAY}"
fi
# Hack, up the log level to 6: "info" (default is 4: "warning")
display_alert "Hacking log level in u-boot config" "LOGLEVEL=6 for ${target}" "info"
run_host_command_logged scripts/config --set-val CONFIG_LOGLEVEL 6
fi
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
if [[ "${UBOOT_DEBUGGING}" == "yes" ]]; then
display_alert "Enabling u-boot debugging" "UBOOT_DEBUGGING=yes" "debug"
# Remove unsets...
cp .config .config.pre.debug
cat .config.pre.debug | grep -v -e "CONFIG_LOG is not set" -e "CONFIG_ERRNO_STR" > .config
rm .config.pre.debug
# 0 - emergency ; 1 - alert; 2 - critical; 3 - error; 4 - warning; 5 - note; 6 - info; 7 - debug; 8 - debug content; 9 - debug hardware I/O
cat <<- EXTRA_UBOOT_DEBUG_CONFIGS >> .config
CONFIG_LOG=y
CONFIG_LOG_MAX_LEVEL=7
CONFIG_LOG_DEFAULT_LEVEL=7
CONFIG_LOG_CONSOLE=y
CONFIG_SPL_LOG=y
CONFIG_SPL_LOG_MAX_LEVEL=6
CONFIG_SPL_LOG_CONSOLE=y
CONFIG_TPL_LOG=y
CONFIG_TPL_LOG_MAX_LEVEL=6
CONFIG_TPL_LOG_CONSOLE=y
# CONFIG_ERRNO_STR is not set
EXTRA_UBOOT_DEBUG_CONFIGS
run_host_command_logged CCACHE_BASEDIR="$(pwd)" PATH="${toolchain}:${toolchain2}:${PATH}" \
"KCFLAGS=-fdiagnostics-color=always" \
pipetty make "olddefconfig" "CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\""
fi
# cflags will be passed both as CFLAGS, KCFLAGS, and both as make params and as env variables. (some vendor u-boot's are cuckoo)
# boards/families/extensions can customize this via the hook below
local -a uboot_cflags_array=(
"-fdiagnostics-color=always" # color messages
"-Wno-error=maybe-uninitialized"
"-Wno-error=misleading-indentation" # patches have mismatching indentation
"-Wno-error=attributes" # for very old-uboots
"-Wno-error=address-of-packed-member" # for very old-uboots
)
if linux-version compare "${gcc_version_main}" ge "11.0"; then
uboot_cflags_array+=(
"-Wno-error=array-parameter" # very old uboots
)
fi
# Hook time, for extra post-processing
call_extension_method "post_config_uboot_target" <<- 'POST_CONFIG_UBOOT_TARGET'
*allow extensions prepare after configuring but before compiling an u-boot target*
Some u-boot targets require extra configuration or pre-processing before compiling.
Last chance to change .config for u-boot before compiling.
Also the only chance to change the (local) array `uboot_cflags_array`.
POST_CONFIG_UBOOT_TARGET
# make olddefconfig, so changes made in hook above are consolidated
display_alert "${uboot_prefix}Updating u-boot config with olddefconfig" "${version} ${target_make}" "info"
run_host_command_logged CCACHE_BASEDIR="$(pwd)" PATH="${toolchain}:${toolchain2}:${PATH}" \
"KCFLAGS=-fdiagnostics-color=always" \
pipetty make "${CTHREADS}" "olddefconfig" "CROSS_COMPILE=\"${CCACHE} ${UBOOT_COMPILER}\""
if [[ "${UBOOT_CONFIGURE:-"no"}" == "yes" ]]; then
display_alert "Configuring u-boot" "UBOOT_CONFIGURE=yes; experimental" "warn"
run_host_command_dialog make menuconfig
display_alert "Exporting saved config" "UBOOT_CONFIGURE=yes; experimental" "warn"
run_host_command_logged make savedefconfig
run_host_command_logged cp -v defconfig "${DEST}/defconfig-uboot-${BOARD}-${BRANCH}"
# check if we can find configs/${BOOTCONFIG}, and if so, output a diff between that and the new saved defconfig
if [[ -f "configs/${BOOTCONFIG}" ]]; then
display_alert "Diffing ${BOOTCONFIG} and new defconfig" "UBOOT_CONFIGURE=yes; experimental" "warn"
run_host_command_logged diff -u --color=always "configs/${BOOTCONFIG}" "${DEST}/defconfig-uboot-${BOARD}-${BRANCH}" "2>&1" "|| true" # no errors please, all to stdout
fi
display_alert "Exporting saved config (experimental)" "${DEST}/defconfig-uboot-${BOARD}-${BRANCH}" "warn"
return 0 # exit after this
fi
##########################################
# REAL COMPILATION SECTION STARTING HERE #
##########################################
local uboot_cflags="${uboot_cflags_array[*]}"
local ts=${SECONDS}
# Collect make environment variables, similar to 'kernel-make.sh'
uboot_make_envs=(
"CFLAGS='${uboot_cflags}'"
"KCFLAGS='${uboot_cflags}'"
"CCACHE_BASEDIR=$(pwd)"
"PATH=${toolchain}:${toolchain2}:${PATH}"
"PYTHONPATH=\"${PYTHON3_INFO[MODULES_PATH]}:${PYTHONPATH}\"" # Insert the pip modules downloaded by Armbian into PYTHONPATH (needed e.g. for pyelftools)
)
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# workaround when two compilers are needed
cross_compile="CROSS_COMPILE=\"$CCACHE $UBOOT_COMPILER\""
[[ -n $UBOOT_TOOLCHAIN2 ]] && cross_compile="ARMBIAN=foe" # empty parameter is not allowed
display_alert "${uboot_prefix}Compiling u-boot" "${version} ${target_make} with gcc '${gcc_version_main}'" "info"
declare -g if_error_detail_message="${uboot_prefix}Failed to build u-boot ${version} ${target_make}"
do_with_ccache_statistics run_host_command_logged_long_running \
"env" "-i" "${uboot_make_envs[@]}" \
pipetty make "$target_make" "$CTHREADS" "${cross_compile}"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
display_alert "${uboot_prefix}built u-boot target" "${version} in $((SECONDS - ts)) seconds" "info"
# Save a defconfig, as that will be included as reference in the .deb package
# Do not fail here; some very (very!) old u-boots like 2011 do not have 'savedefconfig'
run_host_command_logged "env" "-i" "${uboot_make_envs[@]}" pipetty make savedefconfig "$CTHREADS" "${cross_compile}" ||
display_alert "${uboot_prefix}Failed to save defconfig" "${version} ${target_make}" "warn"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
if [[ $(type -t uboot_custom_postprocess) == function ]]; then
display_alert "${uboot_prefix}Postprocessing u-boot" "${version} ${target_make}"
uboot_custom_postprocess
fi
# Hook time, for extra post-processing
call_extension_method "post_uboot_custom_postprocess" <<- 'POST_UBOOT_CUSTOM_POSTPROCESS'
*allow extensions to do extra u-boot postprocessing, after uboot_custom_postprocess*
For hacking at the produced binaries after u-boot is compiled and post-processed.
POST_UBOOT_CUSTOM_POSTPROCESS
declare -a target_dst_files=() # to be filled by deploy_built_uboot_bins_for_one_target_to_packaging_area
deploy_built_uboot_bins_for_one_target_to_packaging_area # copy according to the target_files
# Include metadata about the build, for reference; use the target_counter as part of filename to avoid overwriting
# .config and defconfig go to ${uboottempdir}/usr/lib/${uboot_name}/u-boot-config-target-${target_counter}
# ${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh has general metadata about the target
cat <<- UBOOT_TARGET_METADATA_SH > "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh"
declare -a UBOOT_TARGET_BINS=(${target_dst_files[@]@Q})
declare UBOOT_TARGET_MAKE=${target_make@Q}
UBOOT_TARGET_METADATA_SH
if [[ -f .config ]]; then # Not all u-boots have .config; some very old did boards.cfg or whatever. Ignore in this case
run_host_command_logged cp -v .config "${uboottempdir}/usr/lib/${uboot_name}/u-boot-config-target-${uboot_target_counter}"
cat <<- UBOOT_TARGET_METADATA_SH >> "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh"
declare UBOOT_TARGET_CONFIG="u-boot-config-target-${uboot_target_counter}"
UBOOT_TARGET_METADATA_SH
fi
if [[ -f defconfig ]]; then # Not all u-boots are capable of savedefconfig, and thus defconfig will not be available
run_host_command_logged cp -v defconfig "${uboottempdir}/usr/lib/${uboot_name}/u-boot-defconfig-target-${uboot_target_counter}"
cat <<- UBOOT_TARGET_METADATA_SH >> "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh"
declare UBOOT_TARGET_DEFCONFIG="u-boot-defconfig-target-${uboot_target_counter}"
UBOOT_TARGET_METADATA_SH
fi
if [[ $DEBUG == yes ]]; then
display_alert "${uboot_prefix}Showing u-boot metadata for target" "${version} ${target_make}" "debug"
run_tool_batcat --file-name "/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh" "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata-target-${uboot_target_counter}.sh"
fi
display_alert "${uboot_prefix}Done with u-boot target" "${version} ${target_make}"
return 0
}
function loop_over_uboot_targets_and_do() {
# Try very hard, to fault even, to avoid using subshells while reading a newline-delimited string.
# Sorry for the juggling with IFS.
declare _old_ifs="${IFS}" _new_ifs=$'\n'
IFS="${_new_ifs}" # split on newlines only
display_alert "Looping over u-boot targets" "'${UBOOT_TARGET_MAP}'" "debug"
declare -i uboot_target_counter=1
# save the current state of nullglob into a variable; don't fail
declare _old_nullglob
_old_nullglob="$(shopt -p nullglob || true)"
display_alert "previous state of nullglob" "'${_old_nullglob}'" "debug"
# disable nullglob; dont fail if already disabled
shopt -u nullglob || true
# store new state; don't fail
declare _new_nullglob
_new_nullglob="$(shopt -p nullglob || true)"
display_alert "new state of nullglob" "'${_new_nullglob}'" "debug"
for target in ${UBOOT_TARGET_MAP}; do
display_alert "Building u-boot target" "'${target}'" "debug"
# reset nullglob to _old_nullglob
eval "${_old_nullglob}"
IFS="${_old_ifs}" # restore for the body of loop
declare -g target uboot_name uboottempdir toolchain version
declare -g uboot_prefix="{u-boot:${uboot_target_counter}} "
declare -g target_make target_patchdir target_files
target_make=$(cut -d';' -f1 <<< "${target}")
target_patchdir=$(cut -d';' -f2 <<< "${target}")
target_files=$(cut -d';' -f3 <<< "${target}")
# Invoke our parameters directly
"$@"
# Increment the counter
uboot_target_counter=$((uboot_target_counter + 1))
IFS="${_new_ifs}" # split on newlines only for rest of loop
done
IFS="${_old_ifs}"
# reset nullglob to _old_nullglob
eval "${_old_nullglob}"
uboot_target_counter=$((uboot_target_counter - 1)) # decrement, as we incremented after the last target
declare -g -i uboot_target_counter="${uboot_target_counter}" # set global, for metadata ("how many targets?")
return 0
}
function deploy_built_uboot_bins_for_one_target_to_packaging_area() {
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
display_alert "${uboot_prefix}Preparing u-boot targets packaging" "${version} ${target_make}"
# copy files to build directory
declare f
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
for f in $target_files; do
declare f_src f_dst
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
f_src=$(cut -d':' -f1 <<< "${f}")
if [[ $f == *:* ]]; then
f_dst=$(cut -d':' -f2 <<< "${f}")
else
f_dst=$(basename "${f_src}")
fi
display_alert "${uboot_prefix}Deploying u-boot binary target" "${version} ${target_make} :: ${f_dst}"
[[ ! -f $f_src ]] && exit_with_error "U-boot artifact not found" "$(basename "${f_src}")"
run_host_command_logged cp -v "${f_src}" "${uboottempdir}/usr/lib/${uboot_name}/${f_dst}"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
#display_alert "Done with binary target" "${version} ${target_make} :: ${f_dst}"
target_dst_files+=("${f_dst}") # for metadata
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
done
return 0
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
}
function compile_uboot() {
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build - artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
2023-02-03 14:36:28 +00:00
: "${artifact_version:?artifact_version is not set}"
display_alert "Compiling u-boot" "BOOTSOURCE: ${BOOTSOURCE}" "debug"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
if [[ -n $BOOTSOURCE ]] && [[ "${BOOTSOURCE}" != "none" ]]; then
display_alert "Extensions: fetch custom uboot" "fetch_custom_uboot" "debug"
call_extension_method "fetch_custom_uboot" <<- 'FETCH_CUSTOM_UBOOT'
*allow extensions to fetch extra uboot sources*
For downstream uboot et al.
This is done after `fetch_from_repo`, but before actually compiling u-boot.
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
FETCH_CUSTOM_UBOOT
2022-10-03 03:31:04 +00:00
fi
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# not optimal, but extra cleaning before overlayfs_wrapper should keep sources directory clean
maybe_make_clean_uboot
2022-10-03 03:31:04 +00:00
if [[ $USE_OVERLAYFS == yes ]]; then
local ubootdir
ubootdir=$(overlayfs_wrapper "wrap" "$SRC/cache/sources/$BOOTSOURCEDIR" "u-boot_${LINUXFAMILY}_${BRANCH}")
else
local ubootdir="$SRC/cache/sources/$BOOTSOURCEDIR"
fi
cd "${ubootdir}" || exit
# read uboot version
local version hash
version=$(grab_version "$ubootdir")
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
hash=$(git --git-dir="$ubootdir"/.git rev-parse HEAD)
2022-10-03 03:31:04 +00:00
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
display_alert "Compiling u-boot" "$version ${ubootdir}" "info"
2022-10-03 03:31:04 +00:00
2022-10-08 11:17:30 +00:00
# build aarch64
if [[ $(dpkg --print-architecture) == amd64 ]]; then
local toolchain
toolchain=$(find_toolchain "$UBOOT_COMPILER" "$UBOOT_USE_GCC")
[[ -z $toolchain ]] && exit_with_error "Could not find required toolchain" "${UBOOT_COMPILER}gcc $UBOOT_USE_GCC"
2022-10-03 03:31:04 +00:00
2022-10-08 11:17:30 +00:00
if [[ -n $UBOOT_TOOLCHAIN2 ]]; then
local toolchain2_type toolchain2_ver toolchain2
toolchain2_type=$(cut -d':' -f1 <<< "${UBOOT_TOOLCHAIN2}")
toolchain2_ver=$(cut -d':' -f2 <<< "${UBOOT_TOOLCHAIN2}")
toolchain2=$(find_toolchain "$toolchain2_type" "$toolchain2_ver")
[[ -z $toolchain2 ]] && exit_with_error "Could not find required toolchain" "${toolchain2_type}gcc $toolchain2_ver"
fi
# build aarch64
fi
2022-10-03 03:31:04 +00:00
declare gcc_version_main
gcc_version_main="$(eval env PATH="${toolchain}:${toolchain2}:${PATH}" "${UBOOT_COMPILER}gcc" -dumpfullversion -dumpversion)"
display_alert "Compiler version" "${UBOOT_COMPILER}gcc '${gcc_version_main}'" "info"
[[ -n $toolchain2 ]] && display_alert "Additional compiler version" "${toolchain2_type}gcc $(eval env PATH="${toolchain}:${toolchain2}:${PATH}" "${toolchain2_type}gcc" -dumpfullversion -dumpversion)" "info"
2022-10-03 03:31:04 +00:00
local uboot_name="linux-u-boot-${BRANCH}-${BOARD}"
2022-10-03 03:31:04 +00:00
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# create directory structure for the .deb package
declare cleanup_id="" uboottempdir=""
prepare_temp_dir_in_workdir_and_schedule_cleanup "uboot" cleanup_id uboottempdir # namerefs
mkdir -p "$uboottempdir/usr/lib/u-boot" "$uboottempdir/usr/lib/$uboot_name" "$uboottempdir/DEBIAN"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# Allow extension-based u-boot bulding. We call the hook, and if EXTENSION_BUILT_UBOOT="yes" afterwards, we skip our own compilation.
# This is to make it easy to build vendor/downstream uboot with their own quirks.
display_alert "Extensions: build custom uboot" "build_custom_uboot" "debug"
call_extension_method "build_custom_uboot" <<- 'BUILD_CUSTOM_UBOOT'
*allow extensions to build their own uboot*
For downstream uboot et al.
Set \`EXTENSION_BUILT_UBOOT=yes\` to then skip the normal compilation.
BUILD_CUSTOM_UBOOT
if [[ "${EXTENSION_BUILT_UBOOT}" != "yes" ]]; then
loop_over_uboot_targets_and_do compile_uboot_target
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
else
display_alert "Extensions: custom uboot built by extension" "not building regular uboot" "debug"
fi
if [[ "${ARTIFACT_WILL_NOT_BUILD:-"no"}" == "yes" ]]; then
display_alert "Extensions: artifact will not build" "not building regular uboot" "debug"
return 0
fi
display_alert "Preparing u-boot general packaging" "${version} ${target_make}"
2022-10-03 03:31:04 +00:00
2024-01-28 18:06:17 +00:00
local -a postinst_functions=()
local destination=$uboottempdir
call_extension_method "pre_package_uboot_image" <<- 'PRE_PACKAGE_UBOOT_IMAGE'
*allow making some last minute changes before u-boot is packaged*
This should be implemented by the config to tweak the uboot package, after the board or family has had the chance to.
You can write to `$destination` here and it will be packaged.
You can also append to the `postinst_functions` array, and the _content_ of those functions will be added to the postinst script.
PRE_PACKAGE_UBOOT_IMAGE
artifact_package_hook_helper_board_side_functions "postinst" uboot_postinst_base "${postinst_functions[@]}"
unset uboot_postinst_base postinst_functions destination
2022-10-03 03:31:04 +00:00
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
# declare -f on non-defined function does not do anything (but exits with errors, so ignore them with "|| true")
cat <<- EOF > "${uboottempdir}/usr/lib/u-boot/platform_install.sh"
# Armbian u-boot install script for linux-u-boot-${BOARD}-${BRANCH} ${artifact_version}
# This file provides functions for deploying u-boot to a block device.
2022-10-08 11:17:30 +00:00
DIR=/usr/lib/$uboot_name
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
$(declare -f write_uboot_platform || true)
$(declare -f write_uboot_platform_mtd || true)
$(declare -f setup_write_uboot_platform || true)
2022-10-03 03:31:04 +00:00
EOF
if [[ "${SHOW_DEBUG}" == "yes" ]]; then
display_alert "Showing contents of" "usr/lib/u-boot/platform_install.sh" "info"
run_tool_batcat --file-name "usr/lib/u-boot/platform_install.sh" "${uboottempdir}/usr/lib/u-boot/platform_install.sh"
fi
# Write general metadata. This is intended to be used board-side, and allows some better gui and reuse.
# Ensure that all variables used here are hashed in the artifact-uboot.sh during version calculation.
cat <<- UBOOT_GENERAL_METADATA_SH > "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata.sh"
declare -i UBOOT_NUM_TARGETS=${uboot_target_counter}
declare UBOOT_BIN_DIR="/usr/lib/${uboot_name}"
declare UBOOT_VERSION="${version}"
declare UBOOT_ARTIFACT_VERSION="${artifact_version}"
declare UBOOT_GIT_REVISION="${hash}"
declare UBOOT_GIT_SOURCE="${BOOTSOURCE}"
declare UBOOT_GIT_BRANCH="${BOOTBRANCH}"
declare UBOOT_GIT_PATCHDIR="${BOOTPATCHDIR}"
declare UBOOT_PARTITION_TYPE="${IMAGE_PARTITION_TABLE}"
declare UBOOT_KERNEL_DTB="${BOOT_FDT_FILE}"
declare UBOOT_KERNEL_SERIALCON="${SERIALCON}"
declare UBOOT_EXTLINUX_PREFER="${SRC_EXTLINUX:-"no"}"
declare UBOOT_EXTLINUX_CMDLINE="${SRC_CMDLINE}"
UBOOT_GENERAL_METADATA_SH
if [[ $DEBUG == yes ]]; then
display_alert "${uboot_prefix}Showing u-boot metadata for target" "${version} ${target_make}" "debug"
run_tool_batcat --file-name "/usr/lib/${uboot_name}/u-boot-metadata.sh" "${uboottempdir}/usr/lib/${uboot_name}/u-boot-metadata.sh"
fi
display_alert "Running shellcheck" "usr/lib/u-boot/platform_install.sh" "info"
shellcheck_debian_control_scripts "${uboottempdir}/usr/lib/u-boot/platform_install.sh"
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build - artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
2023-02-03 14:36:28 +00:00
display_alert "Das U-Boot .deb package version" "${artifact_version}" "info"
2022-10-03 03:31:04 +00:00
# set up control file
cat <<- EOF > "$uboottempdir/DEBIAN/control"
2022-10-08 11:17:30 +00:00
Package: linux-u-boot-${BOARD}-${BRANCH}
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build - artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
2023-02-03 14:36:28 +00:00
Version: ${artifact_version}
2022-10-08 11:17:30 +00:00
Architecture: $ARCH
Maintainer: $MAINTAINER <$MAINTAINERMAIL>
Section: kernel
Priority: optional
Provides: armbian-u-boot
Replaces: armbian-u-boot
Conflicts: armbian-u-boot, u-boot-sunxi
hashed-OCI-revisioned-debs: introduce "reversioning" of .deb packages > tl-dr: > - maximize OCI cache hit ratio across nightlies/releases/PRs/etc; > - publish simple `Version:`'s that don't include a crazy hash in repo and images > - introduce `output/packages-hashed` directory > - radically change the `output/debs` directory structure - simplify artifact's `prepare_version()` method for `deb` and `deb-tar` artifacts: - `artifact_base_dir` and `artifact_final_file` will now be auto-calculated; thus removed from each artifact (except `rootfs`) - `artifact_deb_repo` ("global", "jammy", "bookworm") is now required; "global" means common across all RELEASES - `artifact_deb_arch` is now required, "all" is arch-independent, otherwise use `${ARCH}` - `artifact_map_debs` is now auto-calculated based on the above, and shouldn't be specified manually - `artifact_final_version_reversioned` is optional, and can force the final version of the artifact (specific for the `base-files` case) - artifacts that need special handling for reversioning can add function names to `artifact_debs_reversion_functions` array (`base-files` and `bsp-cli` cases) - artifacts `prepare_version()` should set `artifact_version`, but _never_ include it in other variables; `artifact_version` is now changed by framework after `prepare_version()` returns - no longer use/refer/mention `${REVISION}` when building packages. All packages should be `${REVISION}`-agnostic. - `${REVISION}` (actually, `artifact_final_version_reversioned`) will be automatically swapped in the `control` file during reversioning - `fakeroot_dpkg_deb_build()` now takes exactly two arguments: the directory to pack, and the deb ID (key of `artifact_map_packages` dict); add this change in all the artifact's code for this - `obtain_complete_artifact()`: - automatically adds `-Rxxxx` "revisioning-hash" to `artifact_version`, by hashing the revisioning functions and any `artifact_debs_reversion_functions` set - calculates more complex subdirectory paths for both the `output/packages-hashed` and `output/debs`/`output/debs-beta` directories - with the new subdirectories we can be sure a re-version is already done correctly and can skip it (eg, for partial `download-debs` re-runs) - in the future we can automatically clean/remove old versions that are no longer relevant based on the dir structure - exports a lot more information to JSON, including the new subdirectory paths - comment-out code that implemented `skip_unpack_if_found_in_caches`, I'm very unsure why we had this in the first place - `obtain_artifact_from_remote_cache()` - for `deb` type artifacts, OCI won't preserve the subdirectory structure, so move downloaded files to the correct subdirectory manually - this is not needed for `deb-tar`, since that can preserve the dir structure itself - introduce `artifacts-reversion.sh` and its main function `artifact_reversion_for_deployment()` - this has the logic for reversioning .deb's, by `ar`-unpacking them, changing `control.tar` (and possibly `data.tar`), handling `.xz` compression, etc. - also handles hashing those functions, for consistency. Any changes in reversioning code actually change the artifact itself so we're not caught by surprise - by default, it changes `control` file only: - replace `Version:` (which is the hash-version originally) with `artifact_final_version_reversioned` (which is mostly just `${REVISION}`) - add a custom field `Armbian-Original-Hash:` with the original hash-version - `artifact_reversion_for_deployment()` is called by - new CLI wrapper `cli_obtain_complete_artifact()`, used for CLI building of specific artifact, but also for `download-artifact` - `build_artifact_for_image()` used during image build - `armbian-bsp-cli-deb.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `artifact-armbian-base-files.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `kernel`: - add some custom fields to `DEBIAN/control`: - `Armbian-Kernel-Version:` / `Armbian-Kernel-Version-Family:` (for future use: cleanup of usage of `Source: ` field which should be removed) - declutter the `Description:` field, moving long description out of the first line - obtain `IMAGE_INSTALLED_KERNEL_VERSION` from the reversioned deb (this is still a hack and has not been fixed) - `uboot`: - declutter the `Description:` field, moving long description out of the first line - use the reversioned .deb when deploying u-boot to the image - `main_default_build_packages()` now stores reversioned values and complete paths to reversioned .deb's - `list_installed_packages()` now compares custom field `Armbian-Original-Hash: `, and not the `Version:` to make sure debs in the image are the ones we want - `install_artifact_deb_chroot()` is a new wrapper around `install_deb_chroot()` for easy handling of reversioned debs - use it everywhere `install_deb_chroot()` was used in `distro-agnostic.sh` and `distro-specific.sh`
2023-08-02 14:45:16 +00:00
Description: Das U-Boot for ${BOARD}
${artifact_version_reason:-"${version}"}
2022-10-03 03:31:04 +00:00
EOF
# copy license files, config, etc.
[[ -f .config && -n $BOOTCONFIG ]] && run_host_command_logged cp .config "$uboottempdir/usr/lib/u-boot/${BOOTCONFIG}" # legacy and @TODO should be removed as it has only the last target; we now have per-target configs and defconfigs
[[ -f COPYING ]] && run_host_command_logged cp COPYING "$uboottempdir/usr/lib/u-boot/LICENSE"
[[ -f Licenses/README ]] && run_host_command_logged cp Licenses/README "$uboottempdir/usr/lib/u-boot/LICENSE"
[[ -n $atftempdir && -f $atftempdir/license.md ]] && run_host_command_logged cp "${atftempdir}/license.md" "$uboottempdir/usr/lib/u-boot/LICENSE.atf"
2022-10-03 03:31:04 +00:00
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build - artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
2023-02-03 14:36:28 +00:00
display_alert "Building u-boot deb" "(version: ${artifact_version})"
dpkg_deb_build "$uboottempdir" "uboot"
2022-10-03 03:31:04 +00:00
armbian-next: artifacts: firmware/rootfs and actual usage of artifacts during image build - artifacts: introduce `ARTIFACT_IGNORE_CACHE=yes` - artifacts: introduce `DONT_BUILD_ARTIFACTS`, list of artifacts that if not found cached, fail the build - kernel_package_source() is no more - a long dissertation about kernels, families, and the universe - artifacts: actually use rootfs artifact for image build - artifacts: detangle via artifact_base_dir - artifacts: rootfs: use folders in artifact_name; include cache_type - artifacts: some cleanups / validations - rootfs artifact; drop old rootfs cli - artifacts: new CLI shortcuts; remove old firmware CLI - artifacts: full firmware & usage - use firmware artifacts in image build and install its debs - kernel artifact sans legacy; use tmpdir for .deb target for all packages - legacy artifact versions is no more; pack/unpack now in common obtain; - artifacts: uboot: cleanup legacy renaming, use artifact version directly - artifacts: add firmware (small) artifact - deploy uboot to loop from artifact; allow tty to artifact; todos for cleaning - fixes, kernel dtb/headers conditional; remove `.git` from Makefile url; use mapfile for finding files to hash - completely remove KERNEL_HAS_WORKING_HEADERS_FULL_SOURCE and `kernel_package_callback_linux_headers_full_source()` - don't use realpath for artifact_file_relative - curb some warnings - fix: only install headers & dtbs if such exist - kernel .config hook modification hash versioning - OCI_TARGET_BASE vs per-artifact defaults; only deploy to remote from CLI with OTB - artifact revolver & installing into image - add artifact_map_packages and artifact_map_debs dicts - revolver accumulates all info - REPOSITORY_INSTALL is no more (for uboot/kernel, later others) - rename `VER` to `IMAGE_INSTALLED_KERNEL_VERSION`
2023-02-03 14:36:28 +00:00
[[ -n $atftempdir ]] && rm -rf "${atftempdir:?}" # @TODO: intricate cleanup; u-boot's pkg uses ATF's tempdir...
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
done_with_temp_dir "${cleanup_id}" # changes cwd to "${SRC}" and fires the cleanup function early
display_alert "Built u-boot deb OK" "linux-u-boot-${BOARD}-${BRANCH} ${artifact_version}" "info"
armbian-next: `lib` changes - MEGASQUASH - squashed changes from c9cf3fc241cfb4c872f4aef7bbc41d5854db7ea3 to 6809de3d6063cb041205a8318e19da6a4dee68c9 ref extensions_08_10_2022_pre_v30 - also compile.sh - shellfmt lib - split off shell and python tools under lib - revert removal of stuff a-n no longer uses (ref. compilation): general packaging, mkdeb etc - editoconfig split off - extension changes split off - sources and sources/families split off - some undue stuff removed or split armbian-next: manual merge (30) of lib changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (30) of family/board changes between 882f995e21f4791d773bc59efdfdb676494ee6ba and 31ac6383e1ac7ebddd0813abc0f1f9632a9c9c40 armbian-next: manual merge (29) of family/board changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (29) of lib changes between 3435c4636723e9553be0a9ce3666bddb0887069e and 882f995e21f4791d773bc59efdfdb676494ee6ba (A LOT!) armbian-next: manual merge (28) of lib changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (28) of sources/families changes between revisions af6ceee6c55bd4726139fb8b1ad77641d385515a and 38df56fbf39739b48b31b5e393f15261d053b3cd armbian-next: manual merge (27) of `lib` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: manual merge (27) of `sources/families` changes between revisions 9c52562176390624d33c93634e63a8f91cc0815b and af6ceee6c55bd4726139fb8b1ad77641d385515a armbian-next: move `ROOTFSCACHE_VERSION` resolution from GitHub from `main-config` down to `create-cache` - this way config does not depend on remote... armbian-next: move `ARMBIAN_MIRROR` selection (network) from `main-config` to `prepare-host` - this way CONFIG_DEFS_ONLY can run without touching the network armbian-next: manual merge (26) of MD5-checking via debsums (3955) re-imagined - @TODO make sure armbian-next: manual merge (26) of sources/families changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b armbian-next: manual merge (26) of lib changes between revisions 20ee8c54502f877bfa563e113d67709cb80dc6f9 and 9c52562176390624d33c93634e63a8f91cc0815b - @TODO NOT including the md5/debsums check, that needs further rewrite armbian-next: manual merge (25) of lib changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 - @TODO hmm Igor is now going out to the network for rootfs cache version during configuration phase!!! BAD BAD BAD armbian-next: manual merge (25) of family changes between revisions fe972621c61f339c9b84aaf94fd851fa789882e8 and 20ee8c54502f877bfa563e113d67709cb80dc6f9 armbian-next: manual merge (24) of families changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (24) of lib changes between revisions 9ca9120420aa2a39b91ab7283a7dafe113de42e8 and 560531a63505eb610d269b26858689307a9483f5 armbian-next: manual merge (23) of all changes between revisions 17b4fb913c76233d1e2126cfd35ecb85acfa5a25 and 9ca9120420aa2a39b91ab7283a7dafe113de42e8 armbian-next: manual merge (22) of all changes between revisions 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 and 1dddf78cd0ed096edd6506241cfd9aa8cc27ba4e - @TODO EXCEPT the insanity about locales/eval/VERYSILENT in #3850, requires deep review armbian-next: manual merge (21) of all changes between revisions e7d7dab1bb50c1172206629a44f983c1794f8359 and 0eb8fe7497aebf341bead2b3b0fb7309fd7bdc14 armbian-next: fix: patching CREATE_PATCHES=yes - needed to create output dir armbian-next: add `python2-dev` dep for old uboots - cleanup some comments armbian-next: manual merge (20) of all changes between revisions 6b72ae3c864fc020c6aadea473997ca033a0d8cc and 247c4c45fd18aa370575998443fc837466c02971 armbian-next: fix: pass `TERM` to kernel's make, so `make menuconfig` can work armbian-next: fix: git: read commit UNIX timestamp/local date correctly - `checked_out_revision_ts` was correct; git outputs `%ct` as a UNIX timestamp, UTC-based - `checked_out_revision_mtime` was incorrect: git output it without converting to local time - manually convert using `date @xx` so it has correct local time, whatever it is. - add debugging to `get_file_modification_time()` too armbian-next: abstract `$QEMU_BINARY` to `qemu-static.sh`: `deploy_qemu_binary_to_chroot()`/`undeploy_qemu_binary_from_chroot()` - add hackish logic to avoid removing binary that would be needed if image actually contains `qemu-user-static` package armbian-next: fix `uuidgen` basic dep check; use fake bash `$RANDOM` if uuidgen not available - not good: we need uuidgen to begin logging, but it may not be installed yet. workaround. armbian-next: retry 3 times download-only also for `PACKAGE_LIST_BOARD` - acng is really not helping armbian-next: allow customizing UBUNTU_MIRROR (ports mirror) with `CUSTOM_UBUNTU_MIRROR_ARM64=host/path` armbian-next: WiP: kernel make via `env -i` for clean env; show produced /boot tree armbian-next: manual merge (19) of all changes between revisions b23498b94909a855d8b0fda1a95ccfc0ab17c747 and e621d25adc8ee66b0d14c94605ef462a13464b85 - the ssh firstrun revert stuff mostly armbian-next: *breaking change* remove `LIB_TAG` and `.ignore_changes` completely - one day should be replaced with an "update checker" extension, or even "update-enforcer" - for now this just causes chaos armbian-next: `python2` is required for some u-boot builds - would be "use `python-is-python2` so /usr/bin/python exists and points to Python 2.x" but Jammy does not have that anymore - python2 is required for some u-boot builds. - that said, python 2.x is deprecated for a while and needs work thus @TODO armbian-next: bump Python info gatherer to RELEASE=jammy too armbian-next: add `KERNEL_MAJOR_MINOR` info to `media` kernel (@balbes150) - 5.18 is not yet released so might be a problem here armbian-next: allow to skip submodules during `fetch_from_repo`; introduce hook `fetch_custom_uboot` - via GIT_SKIP_SUBMODULES=yes, which disables all submodules everywhere - via UBOOT_GIT_SKIP_SUBMODULES=yes, which disables fetching of submodules during uboot fetch (hidden rkbins anyone?) - extension hook `fetch_custom_uboot` so we can fetch our own stuff if needed armbian-next: `initrd` caching fixes (always enable hook; if cache hit, convert to uImage too) armbian-next: introduce `initramfs`/`initrd` caching - using hashes of (hopefully) all involved files - cache hits are rewarded with sprinkly hearts.   - why? this proves we got a reproducible kernel modules build!   - also, you just saved yourself 2-10 minutes of pain armbian-next: manual merge (18) of changes between revisions 08cf31de73de5f5ba1986348e4556b00d77af09f and c8855aa08dafaa02c939039e55e01967bad61c28 - heh; most bash code changes are for things already done in -next, or no longer used - some version bumps, etc armbian-next: cleanup entrypoint and shuffle `prepare_host_basic()` into logging section armbian-next: *breaking change* add global extlinux killswitch `ALLOW_EXTLINUX` - unless you set `ALLOW_EXTLINUX=yes`, then `SRC_EXTLINUX` will be disabled globally. - add a bunch of logging regarding extlinux, armbianEnv and bootscripts for clarity during build - this is due to nand-sata-install problems with extlinux - some boards _only work_ with extlinux; we'll have to handle it later armbian-next: extensions: `image-output-{qcow2|ovf}`: virtual output formats - which use `qemu-utils` for `qemu-img` conversion of the .img armbian-next: extension: `kernel-localmodconfig`: faster/slimmer kernel builds with `make localmodconfig` armbian-next: extension: `cleanup-space-final-image`: zerofree, slim down firmware, show used space armbian-next: introduce `do_with_ccache_statistics` and use it for kernel compile - some TODOs - better logging for .config copying armbian-next: *breaking change* really disable apt sources for non-desktop builds armbian-next: fix: don't manage apt-cacher-ng if told NOT to, not the other way around armbian-next: `JUST_UBOOT=yes` + hooks `build_custom_uboot()`/`post_write_uboot_platform()` - post_write_uboot_platform()   - only runs during build, for now (not on device) - build_custom_uboot()   - allow fully custom, extension driven, building of u-boot   - also partial preparation of uboot source combined with default Armbian build - HACK: u-boot: downgrade some errors to warnings via KCFLAGS - fix copy of atf bins to uboot, don't do it if atf's not there armbian-next: fix: no use testing the host for resolvconf if we're manipulating the SDCARD armbian-next: sunxi_common: avoid shortcircuit error on family_tweaks_bsp when family_tweaks_bsp_s is not defined armbian-next: fix: add `zstd` and `parallel` to hostdeps armbian-next: manual merge (17) of all changes between revisions 64410fb74b44cca82e9affbdf93a59d5692862ff and 08cf31de73de5f5ba1986348e4556b00d77af09f - changes about `git safe dir` ignored, I've done the same in a different way - hash calculation changes ignored, fasthash is completely different armbian-next: add `crossbuild-essential-armel` so `arm-linux-gnueabi-gcc` is available with system toolchains - need to for some ATF builds, at least. armbian-next: rockchip64_common: lotsa logging and debugging - supposedly no practical changes armbian-next: grub: better logging armbian-next: fix for chaos caused by git's fix of CVE-2022-24765 otherwise "fatal: unsafe repository" - might not be the best solution, but it's the only one I found partitioning: fix: don't try fixing a bootscript that's not there - this fixes a bug when "rootpart=2" without rootpart 1 being /boot armbian-next: cleanups: umount tmpfs-based $SDCARD during cleanup too armbian-next: indented heredoc, no functional changes armbian-next: fix shortcircuit as last statement in case of extlinux - yes, I wasted 3 hours on this tiny bit, so *you* don't have to! - better logging for rootfs `mkfs` et al - introduce `PRESERVE_SDCARD_MOUNT=yes` to preserve SDCARD, MOUNT, and LOOP for debugging armbian-next: kernel-headers: less verbose, trimmed down tools a bit (perf and testing) khadas-vim3l: add asound.state for Khadas VIM3L armbian-next: introduce hook `extension_finish_config()` - late hook for ext configuration - `extension_finish_config()` is the last thing done in config phase     - use it for determining stuff based on kernel version details, package names, etc - also tune down some logging which was too verbose - CI logs with no ANSI escape codes armbian-next: shuffle around code and logic of `add_desktop_package_sources()` - @TODO: still needs proper asset logging for sources.list(.d) - @TODO: tunes down adding of sources/packages to CLI builds, check with Igor armbian-next: 4.x can't build objtool in kernel-headers; allow for handling that later - 4.x has a lot more obtuse dependencies - introduce KERNEL_HAS_WORKING_HEADERS calculated based on KERNEL_MAJOR_MINOR armbian-next: downgrade `error=misleading-indentation` to warning - some 4.x kernels patches are really messy - newer gcc's make that an error now armbian-next: *allow cross compilation*, even the so-called "reverse cross-compile" (amd64 on arm64) armbian-next: add `zfs` extension, which installs headers and builds ZFS via DKMS in chroot - similar to how `nvidia` extension does it armbian-next: x86: enable `nvidia` extension for all releases (only desktop) armbian-next: `headers-debian-byteshift.patch` is dead; long-live cross-compiled source-only kernel-headers - kernel-headers package now only includes _sources_ - postinst does the compilation and preparation for DKMS compatibility - `tools` dir is included now, which includes the byteshift utilities - handle special scripts/module.lds case after 5.10 - tested on a 6 combinations of `x86` / `arm64` / `armhf` (3x targets, 2x hosts) - @TODO: we might be able to reduce the size of tools a bit (perf/tests/etc) - @TODO: still missing ARCH vs ARCHITECTURE vs SRC_ARCH clarity elsewhere armbian-next: allow `use_clean_environment=yes` for `chroot_sdcard_apt_get()` and descendants - this causes command to be run under `env -i`, for a clean environment armbian-next: manual merge (16) of all changes between revisions be9b5156a4bdcb3850fbe0e7ada0f4421acde174 and 2a8e1ecac1c4fdbf986034be9d6c05a8f1b6e6fb - many `traps` ignored: we don't use them anymore armbian-next: fix logging for apt sources/gpg keys armbian-next: don't leak `if_error_xxx` vars across runner helper invocations; always clean then (even if no error) - also: fix wireguard-tools install, had a double parameter there bcm2711: rpi4b: add `pi-bluetooth` which provides working Bluetooth armbian-next: fixes for (non-)logging when interactively configuring kernel (`KERNEL_CONFIGURE=yes`) armbian-next: move `lz4` rootfs caches to `zstd`, multithreaded armbian-next: customize.sh: error handling, do not mount overlay if it doesn't exist armbian-next: extra info for runners; `if_error_detail_message` and `if_error_find_files_sdcard` globals - those are unset after running any command - if error occur, message and/or found files will be included in log, for clarity armbian-next: manual merge (15) of all changes between revisions 0f7200c7932bb04a34601a25fe30dca0197fbc8b and 101eaec90750f23cae3075d6b5329a33b2bf4685 armbian-next: better logging for `rsync` calls everywhere - make rsync verbose armbian-next: downloads: skip download if no `ARMBIAN_MIRROR` nor `DOWNLOAD_MIRROR`; less logs armbian-next: update rockchip.conf from master and use runners armbian-next: update mvebu64.conf from master and use functions armbian-next: git: fix `fetch_from_repo` with actual submodules usage armbian-next: `armbian-next`ify the `nvidia` extension after rebase from master - driver version is configurable via `NVIDIA_DRIVER_VERSION` - use runner function to log/error-handle/use apt cache/etc rpi4b: there's no legacy branch anymore, remove it from KERNEL_TARGET armbian-next: `download_and_verify` non-error handled; logging is messy [WiP] [HACK] armbian-next: logging: let ANSI colors pass to logfile; CALLER_PID instead of BASHPID in subshell armbian-next: enable HTTPS CONNECT in Armbian-managed apt-cacher-ng configuration - PPAs require it armbian-next: don't loop forever if we can't obtain ARMBIAN_MIRROR from redirector - also, don't even try to do it if `SKIP_ARMBIAN_REPO=yes` armbian-next: manual merge (14) of all changes between revisions 13469fd8a9eda4a53f87b7dfee0f431dbab8db54 and 09e416e31cc01ece4533a65f02a470a4c21b90ea - also editorconfig and compile.sh (root) changes armbian-next: *much* improved logging to HTML; log archiving; consistency - keep only current logfile - log to LOGFILE also if SHOW_LOG=yes - log cmd runtime and success/error directly in runner armbian-next: *breaking change* use `MemAvailable` (not including swap) and up requirements for tmpfs - of course add debugging logs - rename vars - should really only use this if we've really a lot of completely free RAM to spare - otherwise OOM killer comes knocking - or swapping to disk, that is counter-productive armbian-next: *breaking change* `DEB_COMPRESS=none` by default if not running in CI/GHA armbian-next: *breaking change* `CLEAN_LEVEL=make` is no more; new `make-kernel`, `make-atf`, `make-uboot` - allows individual control of what to clean - this effectively disables `make clean` by default - rebuilds work and timestamping works for patching, so no reason to clean everytime by default. armbian-next: refactor `prepare_host`, give `apt-cacher-ng` some much needed attention - library dir for host-related stuff, pull it out of "general" finally armbian-next: hostdeps: all toolchains via `crossbuild-essential-arm64`/`armhf`/`amd64` - trying to sort out hostdeps for Jammy [WiP] armbian-next: remove `eatmydata` usage, leftover from failed tries to make git faster armbian-next: fix git origin check, recreate working copy if origin does not match - fix cold bundle https download progress reporting armbian-next: finally consolidating logs into output/logs; colorized HTML logs armbian-next: introduce `do_with_retries()` and use it for apt remote operations during image build armbian-next: another round of logging tuning/fixes; log assets; git logging - introduce `do_with_log_asset()` and `LOG_ASSET=xxx` - separate "git" logging level - add `trap_handler_cleanup_destimg()` to cleanup DESTIMG armbian-next: kernel: use parallel compressors; reproducible kernel builds - also remove leftover mkdebian/builddeb parameters in make invocation - add pbzip2 to hostdeps armbian-next: tuning logging for timestamp/fasthash related stuff which is very verbose - idea is to not completely overwhelm `SHOW_DEBUG=yes` case - make patching quieter and use file instead of stdin - set checked_out_revision_ts during git checkout (timestamp version of _mtime) - timestamp | fasthash logging level (via `SHOW_FASTHASH=yes`) armbian-next: completely remove mkdebian/builddeb/general-packaging kernel packaging stuff armbian-next: manual merge (12) of all changes between revisions 34d4be6b7b346507af58133125cef0bc8ff5aef1 and 5fe0f36aa832ac64d1db05adc4eb09a4df4b5509 armbian-next: introduce `PRESERVE_WORKDIR=yes` for heavy debugging armbian-next: packaging linux-headers again - do NOT use any output from `make headers_install` - that's for libc headers - grabs "headers" (and binary tools) directly from the kernel build tree, not install target - does not produce headers if cross compiling, for now - produces kernel-headers package for the architecture on which it was built - doing a single make invocation with build and install for packaging   - using 'make all' in place of vmlinuz/bzImage/image/zImage armbian-next: apt download-only retried 3 times before installing main packages armbian-next: fix `VER=` read from kernel-image package, also add `linux` default armbian-next: some logging for atf compilation armbian-next: rewrite hostdeps as array, add armhf toolchains armbian-next: distro-agnostic: cleanups armbian-next: armbianEnv hooks/debugs (bsp / image) armbian-next: rpi: completely remove dtb hacks, allowing flash-kernel to work again armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, upgrades untested armbian-next: refactor new kernel packaging; extract hook helper, fix kernel symlink armbian-next: refactor new kernel packaging; add linux-dtb package back in finally, all hooks untested flash-kernel: fix short-circuits as last statement in functions armbian-next: do not force `SHOW_LOG=yes` if `CI=true`; let's _trust_ logging and error handling works armbian-next: back out of setting mtime to the revision time during git checkout. - of course this causes huge recompiles when wanted revision moves forward armbian-next: sync 'config' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 - this is _also_ getting out of hand... gotta merge soon armbian-next: sync 'packages' dir from master revision ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: manual merge (11) of all lib/*.sh changes between revisions 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 and ed589b248aef675cb5e845fcb55c12dfbcc5e925 armbian-next: more refactorings, general logging; fixes; split image stuff - logging flowing correct to LOGDIR, still needs packaging armbian-next: complete removal of usages of `LOG_SUBPATH`; 100% error handled - loose ends, use new LOGDIR - remove the last shortcircuit in extensions execution, now it's 100% error handled - many logging fixes - still missing: final log consolidation/cleanup logging: blue groups if `SHOW_DEBUG=yes` or `SHOW_GROUPS=yes` (console equivalent of CI's avocado) armbian-next: shut down some too-verbose logging: logo building and update-initramfs armbian-next: git/patching, kernel: use date from git as mtime minimum for patched files - use revision's date from git log as mtime for all fetch_from_repo - fix patched files date at least checkout date, otherwise some patches never build armbian-next: first attempt at kernel packaging rework; just linux-image pkg, no dtbs yet - correctly predict KERNELRELEASE, put image-dtbs in the right spot for flash-kernel - remove dpkg-gencontrol, do it all directly armbian-next: rework kernel source packaging, now exporting from git, to .tar.zst - compress with zstdmt (multi-thread zstd), remove pv indicator, it's much faster anyway - export from git (soon will have all patches et al too) - better logging, show pkg name - much, much faster due to zstdmt and deb with none compression armbian-next: a bit atrocious, nameref loaded, `get_list_of_all_buildable_boards()` - in the process, add support for userpatches/config structure mirroring core, for easy upstreaming armbian-next: make `SKIP_EXTERNAL_TOOLCHAINS=yes` default. lets see what breaks [WiP] armbian-next: keeping stdout clean, use display_alert() for cleanup logging armbian-next: library cleanups; remove `build-all.sh` for good; bring `patching.sh` back armbian-next: `interactive_desktop_main_configuration()` and stderr'ed + error handl'ed dialog - use redirection to fd 3 for dialog, now cleanly on stderr - `show_menu()` -> `dialog_menu()` et al - interactive configuration now works again! armbian-next: logging: `SHOW_PIDS=yes` armbian-next: refactor and error-handle most of desktop configuration, incl menus/dialog - `dialog_if_terminal_set_vars()` in place of `dialog_if_terminal()` [WiP] ddk stuff, allow if not in `$KERNEL_TARGET` armbian-next: split `compile_kernel()` function into smaller functions (+logging) - `do_with_logging()` galore, much better error reporting for kernel - `do_with_hooks()` is for the future, just a placeholder for now armbian-next: `do_with_hooks()` placeholder for future ideas armbian-next: logging: small refactor and `do_with_logging` admit it does not do error control armbian-next: fix: traps: `trap_manager_error_handled` is integer (`-i`) not array (`-a`) armbian-next: sunxi-tools: fix logging for sunxi-tools compilation armbian-next: runners now run bash with `-o pipefail` in addition to `-e` - attention, only affects stuff run through the functions in runners.sh armbian-next: kernel: reduce logging clutter (CC,LD,AR) - hide fasthash_debug under `SHOW_FASTHASH` armbian-next: `armhf` should make `zImage` -- or should it? armbian-next: show logs through ccze; avoid ANSI escapes in file; `SHOW_xxx` control - `SHOW_DEBUG` shows the debug level - `SHOW_COMMAND` shows all invoked commands through the runners - `SHOW_TRAPS` to show 'cleanup' and 'trap' level - `SHOW_TIMING` to show $SECONDS but pretty - replace hardcoded traps/cleanups logging armbian-next: add `KERNEL_MAJOR_MINOR=x.z` to every family, manually from the `KERNELBRANCH` armbian-next: cold/warm bundles import/download/export for fetch_from_repo - warm remote, if present, can be exported shallow - if warm remote bundle is present, can be imported shallow too - fallback to cold bundle if warm not present - export (from cold, if exists + warm) shallow bundle - use temp clone and DATE (not rev or tag) for shallowing, WORKS! - info JSON/CSV, include "config_ok" true/false, kernel/uboot info   - include logs for failed configs too   - core reports ARMBIAN_WILL_BUILD_KERNEL and ARMBIAN_WILL_BUILD_UBOOT now with package names armbian-next: `KERNELDIR` is out, `KERNEL_MAJOR_MINOR` is in for all `meson64`, `rpi4b` and `uefi` armbian-next: new kernel workdir layout: cache/sources/kernel/<ARCH>-<KERNEL_MAJOR_MINOR>-<LINUXFAMILY> - `GIT_FIXED_WORKDIR` is used to ignore 2nd param and use a specific dir - this now REQUIRES `KERNEL_MAJOR_MINOR` to be set. - prepare some `WARM_REMOTE_NAME` and related, based on it armbian-next: JUST_KERNEL=yes (opposed to KERNEL_ONLY=yes) is really just the kernel build armbian-next: fetch_from_repos now done when actually compiling atf/uboot/kernel, not before - lib regen after removing empty files (sources.sh and errors.sh are now gone) armbian-next: linux: back to Torvalds bundle, no tags; reminder about export armbian-next: full cached kernel build; refactor all make's in a single place, even for packaging - 2nd+ runs build in less than a minute - kernel: compile and package in a single step, more efficient? - KERNEL_BUILD_DTBS yes/no to build or not dtbs, replaces KERNEL_EXTRA_TARGETS - dtbs_install, modules_install and headers_install now called by Armbian, not packaging - kernel with split, but identical, build and install makes for modules/headers/dtbs - make mkdebian and builddeb idempotent as possible - keep a lot more cache, specially 'debian' folder - filtering logging of install stuff - might be a few leftovers, revisit gains with packaging later   - keeping the arm64 makefile Image.gz vs Image hack   - fix order of packaging patch byteshift, but still there   - cleaning of scripts tools on cross compile removed (!) armbian-next: minor stylistic changes that I end up doing while working on other stuff - I am `OFFLINE_WORK`'ing, I don't wanna waste 3 seconds, thanks - OCD-fix of double `local` declarations [giga-wip] rework patching, introducing fasthash [wip] git: experiment with stable kernel bundle, and all tags. nice, but for what? - also: never delete working copy, exit with error instead. [wip] disable make clean during packaging. I wanna rebuild fast, always [NO PR?] armbian-next: export CHOSEN_KERNEL_WITH_ARCH for reporting info - fix info gathering, parse all boards first, and stop if some failed - fix KERNEL_TARGET regex by including optional "export " - add export from info to CSV, very basic stuff, but works [squash] remove ddk bullshit from KERNEL_TARGET armbian-next: remove file logging of aggregation stuff. config phase can't touch disk anymore. [WiP] git cold bundle; no: shallow clones/fetched; yes: locally packed repo armbian-next: reorder functions in file, they have a ~logical call-tree order armbian-next: move `fingerprint_image()` out of `git.sh` into its own file logging: fix for double stderr redirect during `fakeroot_dpkg_deb_build` logging: subdued "command" logging for debugging low level cmd invocations ("frog") armbian-next: when showing log, emit all host-side invocations as `debug` too. [WiP] trap handler abstraction, somewhat works! armbian-next: manual merge (10) of all lib/*.sh changes between revisions a4ae3a22703a66f1e383a193d74655533ff96c3a and 3305d45b8175dbdd647dc1dc53fa6f0a522920c7 - but NOT the git unshallow stuff, that will be rewritten armbian-next: trapmanager pt1, identifying spots for trap manager intervention armbian-next: `install_pkg_deb` -> `install_host_side_packages` which is completely rewritten version - much simplified; compare installed packages vs wanted, and only update if some missing armbian-next: force u-boot and kernel's gcc to output colors to make easy to spot warnings and errors docker: pass the `CI` env var through Docker invocation, for GitHub Actions armbian-next: avoid warning if "file" utility not installed - should not happen, but better safe than sorry armbian-next: disable long_running `pv` progress bar for custom case too - will rework later, for now pipe causes subshell and caos armbian-next: if `CI=true` then `SHOW_LOG=yes` always docker: add arm64 toolchain to Dockerfile; warn, but don't break, on modprobe failure armbian-next: docker: use ubuntu:rolling, fix deps, use `$SRC/cache` as container's cache dir armbian-next: logging fixes (padding, don't show error more than once, don't remove trap) armbian-next: fixes for early error handling and logging - split stacktrace-related functions into their own lib file - simplify the traps - some stacktrace formatting for readability armbian-next: fix: don't `trap` `ERR` twice, it causes bash to go bananas regarding `caller` armbian-next: `UPSTEM_VER` -> `UBOOT_REPO_VERSION` and related fixes armbian-next: oops, fix some non-lib changes I missed, up to revision ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: manual merge (9) of all lib/*.sh changes between revisions 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 and ff4346c4682e260be3137a2301ac7a69b97c044d armbian-next: more error handling fixes. avoid shortcircuits. - store previous error message in `MSG_IF_ERROR` (still to be actually shown somewhere during error) armbian-next: more error handling fixes. avoid subshells, shortcircuits, and pipes - add `CFLAGS=-fdiagnostics-color=always` to kernel compile; would need also for u-boot soon WiP: indexing JSON into OpenSearch, all-hardcoded version rpi: add DTB symlink in Debian/Ubuntu standard location /lib/firmware/$version/device-tree; remove build-time-only hacks - this allows us to remove the most horrible hack - should allow for correctly working DTB upgrades - should NOT impact other families, although a new symlink will be created, nothing uses it. rpi: fix: flash-kernel fix to ignore kernel 'flavour' for all raspi's armbian-next: don't try to remove packages that are not installed to begin with - much faster - new chroot_sdcard_with_stdout() runner, without bash or any escaping. armbian-next: don't try to enable systemd services for units that don't exist - those might be removed by a bsp extension, so check for existence before trying to enable armbian-next: don't error/warn on failure to enable bootsplash when systemd units missing armbian-next: use indented HEREDOCS for all call_extension_method's armbian-next: manual merge (8) of all lib/*.sh changes between revisions 1d499d9ac282d44d4fdb052e5f64fb902688c18f and 3b7f5b1f3418b7ad7a7c03f2af6d8da66f20bcb9 armbian-next: manual merge (7) of all lib/*.sh changes between revisions d885bfc97d908b09dddac06393c2987995704d0a and 1d499d9ac282d44d4fdb052e5f64fb902688c18f armbian-next: manual merge (6) of all lib/*.sh changes between revisions c7f3c239fe93f8fcfdd1b1edf6f11e143802e379 and d885bfc97d908b09dddac06393c2987995704d0a armbian-next: avoid writing to disk during configuration; `ANSI_COLOR=none` logging; make CONFIG_DEFS_ONLY=yes runnable without sudo - when `CONFIG_DEFS_ONLY=yes`, avoid writing the config summary output.log file.   - refactor that into a function as to be easy to if-out-of   - don't write to disk during aggregate_content() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during show_checklist_variables() if `CONFIG_DEFS_ONLY=yes`   - don't write to disk during write_deboostrap_list_debug_log() if `CONFIG_DEFS_ONLY=yes`   - don't compress and rotate logs if `CONFIG_DEFS_ONLY=yes` - don't pretend to be handling errors we can't handle during var capture - I foresee a world we can build all .debs without sudo - and a some kind of split of codebase entrypoint due to that future feature - some python info.py enhancements, not ready yet armbian-next: shellfmt and regen library (after rebase from master n.5) tools/shellfmt.sh: exclude "cache" and ".tmp" from formatting, for obvious reasons tools/gen-library.sh: sort function files, so it does not keep changing between runs on different machines. - order should not be important, since files only contain functions, but avoid git churn armbian-next: manual merge (5) of all lib/*.sh changes between revisions 1b18df3c834c5c4166e99f731d53ec88d14caf7d and e7962bb2b555d3afdca0ff168982cc45235ca0a8 - most PKG_PREFIX work was already done armbian-next: `TMPDIR` for all, many logging fixes, error handling: leave-no-garbage-behind without needing traps. - set `MOUNT_UUID` and `WORKDIR`/`MOUNT`/`SDCARD`/`EXTENSION_MANAGER_TMP_DIR`/`DESTIMG` early in do_main_configuration()   - but, they're just _set_ there, dirs are not created early, but on demand later   - still @TODO: actually clean those during error trap. (unhappy path leaves garbage still)   - but does not leave garbage behind during "successful" runs at least (happy path works) - actually export `TMPDIR` (== `WORKDIR`) during start of build (not config!), so all `mktemp` are subject to it   - `runners.sh` has helpers to avoid passing `TMPDIR` to chroot. Use the helpers! don't call `chroot` directly.   - don't trap/cleanup individual `mktemp` temp dirs during .deb packaging's, all is handled at once now.   - kernel packaging, for example, automatically picks up `TMPDIR` too. So now hosts `/tmp` is mostly left alone. - fix some "infodumps" that are done into `.log` files directly. - don't use sudo if `CONFIG_DEFS_ONLY=yes`; we'll only be collecting info, not doing anything. - simpler logging for `rsync` operations (just dump to stdout, logging will handle it!) - use padded counter for section logfiles, so we know which order things ran. exported as `CURRENT_LOGGING_COUNTER` - no reason to use `apt-get` with `-yqq` anymore, since all logging is handled, so now `-y` by default - desktop: using runners helpers for rootfs-desktop.sh, which should help a lot with acng caching and finding of problems - extensions: correctly cleanup temp stuff, extensions has its own tmp/workdir now, and is always cleaned up at end of build. armbian-next: bye `PKG_PREFIX`, hello `run_host_x86_binary_logged()` wrapper function; better error handling - we've x86-only utilities that might need to be run on non-x86 build machines - previously duplicated logic in PKG_PREFIX variable refactored into logged function - added centralized debug logging - replace all PKG_PREFIX usage with the new wrapper function, which already handles logging and errors.   - mostly FIP tooling invocations   - but also the boot_logo image builder   - wrapper function delegates to common `run_host_command_logged` - wrap other FIP invocations with `run_host_command_logged` too, for tidy logging - avoid using conditionals when invoking functions; that completely disables error handling inside the called function - use explicit bash opts instead of shortcuts like `set -e` - a _lot_ of debug logging added armbian-next: always use UPPERCASE labels for FAT32 UEFI filesystems (rpi4b, uefi-*) armbian-next: shellfmt after rebase onto master armbian-next: manual merge (4) of all lib/*.sh changes between revisions 23afccf56e295610340188dc2613143d5323659e and e610f00bc7fdcab5ba4c648fa21148518b4b767b - plus ooops atf: fix for `set -e` mode; fix CROSS_COMPILE quoting Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: predict the future location of .img file - otherwise it's really unhelpful Signed-off-by: Ricardo Pardini <ricardo@pardini.net> uefi: alias `BRANCH=ddk` to `current`'s `DISTRO_GENERIC_KERNEL=yes` - no real change, just to match rpi4b's BRANCH=style - opens space for Armbian-built `current` soon Signed-off-by: Ricardo Pardini <ricardo@pardini.net> rpi: `legacy`->`ddk` (distro default kernel), remove overclock - common vars in bcm2711.conf moved to top - removed overclock/overvolt that was leftover my old setup - confirmed: works with rpi3b too, should work with CM4/CM3 and others - use valid UPPERCASE FAT label for RPICFG (in place of `rpicfg`) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: shellfmt again after rebase Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: manual merge (3) of all lib/*.sh changes between revisions 1035905760884fbea567717684c38faae3ef28a4 and e4e4ab0791e60e1e7a204dd0409fa29a1cf7a376 - missed non lib change on Several improvements for RPi builds (#3391) - I just realized I will have to drop all non lib changes rockship: fixes for `set -e` mode in rockship armhf family and bsp tweaks Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armhf: enable building armhf targets on amd64 using system toolchains - SKIP_EXTERNAL_TOOLCHAINS=yes on amd64 should use the same system toolchains as an arm64 build Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: better logging about family_tweaks and family_tweaks_bsp Signed-off-by: Ricardo Pardini <ricardo@pardini.net> kernel: unblock cross compilation, warn about headers package Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fixes for sunxi/megous stuff with `set -e` Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: fix shellcheck references generation Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge (2) of all lib/*.sh changes between revisions 117633687e9942ade647a8b7d3f80f01e3145193 and 30830388555e463402161a6e0d6993dd6fb0544e Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: renaming function files a bit more consistently Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: removing leftover empty file after all the moving around Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: really insist on set -e during library loading Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: shellfmt again after rebasing master Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: manual merge of all lib/*.sh changes between revisions f6143eff670d435cc8d15b52335b1b70ccb91e1c and f3388b9aee0d1564a28fbc661c6e7b0415a6dfd0 Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: generic do_capturing_defs wrapper; Python parser - enabled by passing CONFIG_DEFS_ONLY=yes; in this case does not build anything - [WiP] Python3 info reader / matrix expander   - multithreaded version Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: move some interactive parts of config into its own functions - mostly from config-prepare; - there is still a lot of others in main-config Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use chroot_custom for grub and flash-kernel extension logging Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: use line buffering, fix runner output color for GHA Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: wrap dpkg-deb; set TMPDIR (not in chroot); refactor kernel make - And a huge amount of @TODO's added - Add "debug" and "deprecation" `display_alert()` levels - insist that `install_common` is now `install_distribution_agnostic` - unrelated: realtek 8822CS is EXTRAWIFI=yes only now, sorry. - many debug statements for desktop Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: don't bail out on patching error Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: bunch of fixes; no-stdin; traps; better stacks - mostly no-stdin dialog handling (desktop et al) - let ERR trap run together with unmount trap (EXIT etc) Signed-off-by: Ricardo Pardini <ricardo@pardini.net> logging: trap ERR very early, pass-in caller info Signed-off-by: Ricardo Pardini <ricardo@pardini.net> armbian-next: huge refactor, shellfmt, codesplit, logging/error handling - *this commit changes most/all the lines of bash code in armbian/build* - *author is set to IgorPec for historical reasons, rpardini is to blame for the actual changes* - logging: refactorings, pt.4: autogen lib, shellfmt tool, extract cli   - shellfmt: auto-downloader and runner of shellfmt     - darwin/linux     - amd64/arm64     - find ~correct files to format     - run formatting     - check formatting soon   - refactor compile's CLI stuff out into function/cli   - gen, and use genned library with tool - logging: refactoring pt3: HUGE split of everything into everything else   - plus rebasing fixes - logging: refactorings, pt. 2: error handling   - fix: no errors during umount_chroot()   - no progress for CI=true builds   - WiP disable kernel hashing. too crazy.   - a few builds now working with "set -e"   - wtf. or something   - kernel logging and long_running stuff - a mess - needs moving around in files   - rewrite uboot compile loop without using subshells. remove ALL traps. refactor host command   - better logging for u-boot   - more fixes, u-boot   - more fixes for logging et al   - git stuff   - many (many) fixes   - new color scheme   - a monster. make sure unmounted at the end. remove set -e's, to-be-readded.   - remove set -e "for safety reasons"   - more alerts. we gotta know whats failing   - some more logging stuff and fixes for error checking   - more logging and error handling stuff   - fixes; some set -e's   - more logging stuff - logging: refactoring codebase, pt.1: functions-only   - Refactor the hell out of everything into functions   - rename build-all-ng to build-multi; other fixes, extensions init   - slight nudge   - some were already good, like this one.   - syntax fixes   - some need a little nudge   - another clean one   - some just need a better name (and splitting later)   - syntax fixes   - some were already good, like this desktop one   - some were already good, like this other one   - some were already good, like this one.   - debootstrap is gone.   - extract functions from compile.sh   - add logging to main_default_build   - more stuff   - cleanups and refactors of main.sh - logging: first steps - logging: pt. 0: shellfmt everything - add riscv64 do SRC_ARCH/ARCH/ARCHITECTURE mess; add warn
2022-10-09 10:37:11 +00:00
return 0 # success
2022-10-03 03:31:04 +00:00
}
2024-01-28 18:06:17 +00:00
function uboot_postinst_base() {
# Source the armbian-release information file
# shellcheck source=/dev/null
2024-01-28 18:06:17 +00:00
[ -f /etc/armbian-release ] && . /etc/armbian-release
# shellcheck source=/dev/null
2024-01-28 18:06:17 +00:00
source /usr/lib/u-boot/platform_install.sh
if [ "${FORCE_UBOOT_UPDATE:-no}" == "yes" ]; then
#recognize_root
root_uuid=$(sed -e 's/^.*root=//' -e 's/ .*$//' < /proc/cmdline)
root_partition=$(blkid | tr -d '":' | grep "${root_uuid}" | awk '{print $1}')
root_partition_name=$(echo $root_partition | sed 's/\/dev\///g')
root_partition_device_name=$(lsblk -ndo pkname $root_partition)
root_partition_device=/dev/$root_partition_device_name
write_uboot_platform "$DIR" "${root_partition_device}"
sync
fi
}