armbian-build/lib/functions/cli/cli-configdump.sh

67 lines
2.6 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
#
# SPDX-License-Identifier: GPL-2.0
#
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
#
# This file is a part of the Armbian Build Framework
# https://github.com/armbian/build/
function cli_config_dump_json_pre_run() {
declare -g -r CONFIG_DEFS_ONLY='yes' # @TODO: This is actually too late (early optimizations in logging etc), so callers should also set it in the environment when using CLI. sorry.
armbian-next: the great cli entrypoint (+docker) rewrite; introduce `USE_LOCAL_APT_DEB_CACHE` replacing `apt-cacher-ng` - armbian-next: introduce `USE_LOCAL_APT_DEB_CACHE` (default `=yes`) as alternative/in addition to `apt-cacher-ng` (eg, in Docker) - this uses `cache/aptcache/${RELEASE}-${ARCH}` (in the host) for - apt cache, by bind-mounting it to `${SDCARD}/var/cache/apt` in the `chroot_sdcard_apt_get()` runner and its usages - debootstrap, by passing it `--cache-dir` - utility function to help understand what is happening to cache during usage - apt itself mantains this cache, removing old packages when new ones are installed. apt does this _by default_ - introduce `DONT_MAINTAIN_APT_CACHE=yes` to skip out of automatic apt maintenance of apt cache, eg, during `remove`s - don't do `apt clean` and such if using local cache, that would clean the cache, not the chroot - clean up `install_deb_chroot()` a little, find an unrelated bug there - WiP: the great cli entrypoint (+docker) rewrite, Phase 6: relaunching structure; re-pass ARMBIAN_BUILD_UUID; use ARMBIAN_COMMAND for log filename; fix for output/logs dir perms - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 4/x; better logging, check & force `DEST_LANG` - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 3/x; don't write to stderr in generated Dockerfile - it's `drastic red` on non-buildx dockers - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 2/x, logging - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 1/x - source configs in a logging section. - Docker: silent, fast retries to make sure `docker system df` works - shut-up `chown` (no `-v`) output related to `SET_OWNER_TO_UID` - ask user to wait while `DESTIMG` is rsync'ed to `FINALDEST` -- it's potentially very slow - use green apple for Mac logging, instead of red apple which might imply error... - WiP: the great cli entrypoint (+docker) rewrite, Phase 4: run as non-root, maybe-with-Docker - introduce `is_docker_ready_to_go()`; if it is, and we're not root, use Docker instead of sudo. <- GOOD IDEA? BAD IDEA? lol - introduce `SET_OWNER_TO_UID` var to be passed to Docker/sudo so written files are owned by the launching user, not root. - introduce `mkdir_recursive_and_set_uid_owner()` and `reset_uid_owner()` to reset owner based on `SET_OWNER_TO_UID` - use it for userpatches files created, logs, and output files, including images and debs. - @TODOs ref. `$SUDO_USER` which I think the old version of this? - add a lot of @TODOs, ref being able to relaunch something that's not `build` inside Docker, also add/change params and configs and command. - initially add `ARMBIAN_DOCKER_RELAUNCH_EXTRA_ARGS` - WiP: the great cli entrypoint (+docker) rewrite, Phase 3: rpardini is demented, v3 - WiP: the great cli entrypoint (+docker) rewrite, Phase 2: rpardini is demented - WiP: the great cli entrypoint (+docker) rewrite, Phase 1 - armbian-next: WiP: Docker: actually use the GHA-image as base; pull it every 24hs. - using image in my private repo. - this has significant speedup to "start building time" on the 1st run - move some Linux specific stuff to its own if - add comments and todo - armbian-next: WiP: Docker, high-WiP, beginnings of Armbian mount dict, with linux/darwin preferences - armbian-next: WiP: Docker, configure `BUILDKIT_COLORS` - armbian-next: WiP: Docker, make docker image from Dockerfile more compact by flattening layers - armbian-next: `logging`: add whale indicator if build running under Docker - armbian-next: WiP: `docker`: working with `bookworm`, `sid`, and `jammy` on Darwin & Linux; works with `bullseye` on Linux only - armbian-next: WiP: `docker`: force ARMBIAN_RUNNING_IN_CONTAINER both in Dockerfile and passed as `--env`; apt update and install in same layer; back to jammy - armbian-next: introduce `armbian_is_running_in_container()` and `armbian_is_host_running_systemd()`, replacing `systemd-detect-virt` in multiple spots - WiP: try with debian:bullseye -- can't detect docker at all - armbian-next: WiP: 2nd stab at new Docker support; Darwin still works; Linux `docker.io` working - gen .dockerignore together with Dockerfile - split in funcs - hacks for Linux and `/dev/loop` stuff, CONTAINER_COMPAT=yes - mac still works, Linux stuff would break it but I if'fed - armbian-next: the secrets of `CONTAINER_COMPAT` revealed; add size checking to check_loop_device() and avoid retry when `mknod`ing - this fails for the right reasons now, causing retries, which are then retried and work ;-) - this is related to building under Docker on Linux, using docker.io package (not docker-ce) - armbian-next: remove `.dockerignore` and add it to `.gitignore`; it's going to be auto-generated - armbian-next: `.dockerignore`: Docker context should only have minimal files and folders, to speed up Dockerfile build - IMPORTANT: `.dockerignore` is going to be generated from now on: so this is the last commit with changes before removal - armbian-next: WiP: initial stab at new Docker support; really run the passed cmdline; add Dockerfile to gitignore - armbian-next: WiP: initial stab at new Docker support; generate Dockerfile; introduce REQUIREMENTS_DEFS_ONLY - uses REQUIREMENTS_DEFS_ONLY - works on Docker Desktop on Mac; - linux TBA - armbian-next: don't error out if `.git` not present; other small fixes - armbian-next: general "work or at least don't misbehave when run on a very bare ubuntu:latest instance" - can't assume things, for example: - that `sudo` will be available; it might not, and might be already root, no reason to fail - that `/etc/timezone` will exist - that `systemd-detect-virt` will be available - that `git` will be available - that `locale-gen` will be available
2022-10-09 15:58:23 +00:00
}
function cli_config_dump_json_run() {
armbian-next: the great cli entrypoint (+docker) rewrite; introduce `USE_LOCAL_APT_DEB_CACHE` replacing `apt-cacher-ng` - armbian-next: introduce `USE_LOCAL_APT_DEB_CACHE` (default `=yes`) as alternative/in addition to `apt-cacher-ng` (eg, in Docker) - this uses `cache/aptcache/${RELEASE}-${ARCH}` (in the host) for - apt cache, by bind-mounting it to `${SDCARD}/var/cache/apt` in the `chroot_sdcard_apt_get()` runner and its usages - debootstrap, by passing it `--cache-dir` - utility function to help understand what is happening to cache during usage - apt itself mantains this cache, removing old packages when new ones are installed. apt does this _by default_ - introduce `DONT_MAINTAIN_APT_CACHE=yes` to skip out of automatic apt maintenance of apt cache, eg, during `remove`s - don't do `apt clean` and such if using local cache, that would clean the cache, not the chroot - clean up `install_deb_chroot()` a little, find an unrelated bug there - WiP: the great cli entrypoint (+docker) rewrite, Phase 6: relaunching structure; re-pass ARMBIAN_BUILD_UUID; use ARMBIAN_COMMAND for log filename; fix for output/logs dir perms - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 4/x; better logging, check & force `DEST_LANG` - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 3/x; don't write to stderr in generated Dockerfile - it's `drastic red` on non-buildx dockers - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 2/x, logging - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 1/x - source configs in a logging section. - Docker: silent, fast retries to make sure `docker system df` works - shut-up `chown` (no `-v`) output related to `SET_OWNER_TO_UID` - ask user to wait while `DESTIMG` is rsync'ed to `FINALDEST` -- it's potentially very slow - use green apple for Mac logging, instead of red apple which might imply error... - WiP: the great cli entrypoint (+docker) rewrite, Phase 4: run as non-root, maybe-with-Docker - introduce `is_docker_ready_to_go()`; if it is, and we're not root, use Docker instead of sudo. <- GOOD IDEA? BAD IDEA? lol - introduce `SET_OWNER_TO_UID` var to be passed to Docker/sudo so written files are owned by the launching user, not root. - introduce `mkdir_recursive_and_set_uid_owner()` and `reset_uid_owner()` to reset owner based on `SET_OWNER_TO_UID` - use it for userpatches files created, logs, and output files, including images and debs. - @TODOs ref. `$SUDO_USER` which I think the old version of this? - add a lot of @TODOs, ref being able to relaunch something that's not `build` inside Docker, also add/change params and configs and command. - initially add `ARMBIAN_DOCKER_RELAUNCH_EXTRA_ARGS` - WiP: the great cli entrypoint (+docker) rewrite, Phase 3: rpardini is demented, v3 - WiP: the great cli entrypoint (+docker) rewrite, Phase 2: rpardini is demented - WiP: the great cli entrypoint (+docker) rewrite, Phase 1 - armbian-next: WiP: Docker: actually use the GHA-image as base; pull it every 24hs. - using image in my private repo. - this has significant speedup to "start building time" on the 1st run - move some Linux specific stuff to its own if - add comments and todo - armbian-next: WiP: Docker, high-WiP, beginnings of Armbian mount dict, with linux/darwin preferences - armbian-next: WiP: Docker, configure `BUILDKIT_COLORS` - armbian-next: WiP: Docker, make docker image from Dockerfile more compact by flattening layers - armbian-next: `logging`: add whale indicator if build running under Docker - armbian-next: WiP: `docker`: working with `bookworm`, `sid`, and `jammy` on Darwin & Linux; works with `bullseye` on Linux only - armbian-next: WiP: `docker`: force ARMBIAN_RUNNING_IN_CONTAINER both in Dockerfile and passed as `--env`; apt update and install in same layer; back to jammy - armbian-next: introduce `armbian_is_running_in_container()` and `armbian_is_host_running_systemd()`, replacing `systemd-detect-virt` in multiple spots - WiP: try with debian:bullseye -- can't detect docker at all - armbian-next: WiP: 2nd stab at new Docker support; Darwin still works; Linux `docker.io` working - gen .dockerignore together with Dockerfile - split in funcs - hacks for Linux and `/dev/loop` stuff, CONTAINER_COMPAT=yes - mac still works, Linux stuff would break it but I if'fed - armbian-next: the secrets of `CONTAINER_COMPAT` revealed; add size checking to check_loop_device() and avoid retry when `mknod`ing - this fails for the right reasons now, causing retries, which are then retried and work ;-) - this is related to building under Docker on Linux, using docker.io package (not docker-ce) - armbian-next: remove `.dockerignore` and add it to `.gitignore`; it's going to be auto-generated - armbian-next: `.dockerignore`: Docker context should only have minimal files and folders, to speed up Dockerfile build - IMPORTANT: `.dockerignore` is going to be generated from now on: so this is the last commit with changes before removal - armbian-next: WiP: initial stab at new Docker support; really run the passed cmdline; add Dockerfile to gitignore - armbian-next: WiP: initial stab at new Docker support; generate Dockerfile; introduce REQUIREMENTS_DEFS_ONLY - uses REQUIREMENTS_DEFS_ONLY - works on Docker Desktop on Mac; - linux TBA - armbian-next: don't error out if `.git` not present; other small fixes - armbian-next: general "work or at least don't misbehave when run on a very bare ubuntu:latest instance" - can't assume things, for example: - that `sudo` will be available; it might not, and might be already root, no reason to fail - that `/etc/timezone` will exist - that `systemd-detect-virt` will be available - that `git` will be available - that `locale-gen` will be available
2022-10-09 15:58:23 +00:00
# configuration etc - it initializes the extension manager
do_capturing_defs config_board_and_remove_useless < /dev/null # this sets CAPTURED_VARS_NAMES and CAPTURED_VARS_ARRAY; the < /dev/null is take away the terminal from stdin
if [[ "${ARMBIAN_COMMAND}" == "config-dump-no-json" ]]; then
# for debugging the bash-declare-to-JSON parser
echo "${CAPTURED_VARS_ARRAY[@]}"
return 0
fi
# convert to JSON, using python helper; each var is passed via a command line argument; that way we avoid newline/nul-char separation issues
display_alert "Dumping JSON" "for ${#CAPTURED_VARS_ARRAY[@]} variables" "ext"
python3 "${SRC}/lib/tools/configdump2json.py" "--args" "${CAPTURED_VARS_ARRAY[@]}" # to stdout
return 0
}
function config_board_and_remove_useless() {
skip_host_config=yes use_board=yes skip_kernel=no do_logging=no prep_conf_main_minimal_ni # avoid logging during configdump; it's useless; skip host config
determine_artifacts_needed_and_its_inputs_for_configdump
# Remove unwanted variables from the config dump JSON.
unset FINALDEST
unset DEB_STORAGE
unset ROOTPWD
armbian-next: the great cli entrypoint (+docker) rewrite; introduce `USE_LOCAL_APT_DEB_CACHE` replacing `apt-cacher-ng` - armbian-next: introduce `USE_LOCAL_APT_DEB_CACHE` (default `=yes`) as alternative/in addition to `apt-cacher-ng` (eg, in Docker) - this uses `cache/aptcache/${RELEASE}-${ARCH}` (in the host) for - apt cache, by bind-mounting it to `${SDCARD}/var/cache/apt` in the `chroot_sdcard_apt_get()` runner and its usages - debootstrap, by passing it `--cache-dir` - utility function to help understand what is happening to cache during usage - apt itself mantains this cache, removing old packages when new ones are installed. apt does this _by default_ - introduce `DONT_MAINTAIN_APT_CACHE=yes` to skip out of automatic apt maintenance of apt cache, eg, during `remove`s - don't do `apt clean` and such if using local cache, that would clean the cache, not the chroot - clean up `install_deb_chroot()` a little, find an unrelated bug there - WiP: the great cli entrypoint (+docker) rewrite, Phase 6: relaunching structure; re-pass ARMBIAN_BUILD_UUID; use ARMBIAN_COMMAND for log filename; fix for output/logs dir perms - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 4/x; better logging, check & force `DEST_LANG` - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 3/x; don't write to stderr in generated Dockerfile - it's `drastic red` on non-buildx dockers - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 2/x, logging - WiP: the great cli entrypoint (+docker) rewrite, Phase 5: cleanups 1/x - source configs in a logging section. - Docker: silent, fast retries to make sure `docker system df` works - shut-up `chown` (no `-v`) output related to `SET_OWNER_TO_UID` - ask user to wait while `DESTIMG` is rsync'ed to `FINALDEST` -- it's potentially very slow - use green apple for Mac logging, instead of red apple which might imply error... - WiP: the great cli entrypoint (+docker) rewrite, Phase 4: run as non-root, maybe-with-Docker - introduce `is_docker_ready_to_go()`; if it is, and we're not root, use Docker instead of sudo. <- GOOD IDEA? BAD IDEA? lol - introduce `SET_OWNER_TO_UID` var to be passed to Docker/sudo so written files are owned by the launching user, not root. - introduce `mkdir_recursive_and_set_uid_owner()` and `reset_uid_owner()` to reset owner based on `SET_OWNER_TO_UID` - use it for userpatches files created, logs, and output files, including images and debs. - @TODOs ref. `$SUDO_USER` which I think the old version of this? - add a lot of @TODOs, ref being able to relaunch something that's not `build` inside Docker, also add/change params and configs and command. - initially add `ARMBIAN_DOCKER_RELAUNCH_EXTRA_ARGS` - WiP: the great cli entrypoint (+docker) rewrite, Phase 3: rpardini is demented, v3 - WiP: the great cli entrypoint (+docker) rewrite, Phase 2: rpardini is demented - WiP: the great cli entrypoint (+docker) rewrite, Phase 1 - armbian-next: WiP: Docker: actually use the GHA-image as base; pull it every 24hs. - using image in my private repo. - this has significant speedup to "start building time" on the 1st run - move some Linux specific stuff to its own if - add comments and todo - armbian-next: WiP: Docker, high-WiP, beginnings of Armbian mount dict, with linux/darwin preferences - armbian-next: WiP: Docker, configure `BUILDKIT_COLORS` - armbian-next: WiP: Docker, make docker image from Dockerfile more compact by flattening layers - armbian-next: `logging`: add whale indicator if build running under Docker - armbian-next: WiP: `docker`: working with `bookworm`, `sid`, and `jammy` on Darwin & Linux; works with `bullseye` on Linux only - armbian-next: WiP: `docker`: force ARMBIAN_RUNNING_IN_CONTAINER both in Dockerfile and passed as `--env`; apt update and install in same layer; back to jammy - armbian-next: introduce `armbian_is_running_in_container()` and `armbian_is_host_running_systemd()`, replacing `systemd-detect-virt` in multiple spots - WiP: try with debian:bullseye -- can't detect docker at all - armbian-next: WiP: 2nd stab at new Docker support; Darwin still works; Linux `docker.io` working - gen .dockerignore together with Dockerfile - split in funcs - hacks for Linux and `/dev/loop` stuff, CONTAINER_COMPAT=yes - mac still works, Linux stuff would break it but I if'fed - armbian-next: the secrets of `CONTAINER_COMPAT` revealed; add size checking to check_loop_device() and avoid retry when `mknod`ing - this fails for the right reasons now, causing retries, which are then retried and work ;-) - this is related to building under Docker on Linux, using docker.io package (not docker-ce) - armbian-next: remove `.dockerignore` and add it to `.gitignore`; it's going to be auto-generated - armbian-next: `.dockerignore`: Docker context should only have minimal files and folders, to speed up Dockerfile build - IMPORTANT: `.dockerignore` is going to be generated from now on: so this is the last commit with changes before removal - armbian-next: WiP: initial stab at new Docker support; really run the passed cmdline; add Dockerfile to gitignore - armbian-next: WiP: initial stab at new Docker support; generate Dockerfile; introduce REQUIREMENTS_DEFS_ONLY - uses REQUIREMENTS_DEFS_ONLY - works on Docker Desktop on Mac; - linux TBA - armbian-next: don't error out if `.git` not present; other small fixes - armbian-next: general "work or at least don't misbehave when run on a very bare ubuntu:latest instance" - can't assume things, for example: - that `sudo` will be available; it might not, and might be already root, no reason to fail - that `/etc/timezone` will exist - that `systemd-detect-virt` will be available - that `git` will be available - that `locale-gen` will be available
2022-10-09 15:58:23 +00:00
}
function determine_artifacts_needed_and_its_inputs_for_configdump() {
# Determine which artifacts to build.
declare -a artifacts_to_build=()
determine_artifacts_to_build_for_image
display_alert "Artifacts to build:" "${artifacts_to_build[*]}" "info"
# For each artifact, get the input variables from each.
declare -a all_wanted_artifact_names=() all_wanted_artifact_vars=()
declare one_artifact one_artifact_package
for one_artifact in "${artifacts_to_build[@]}"; do
declare -g artifact_input_vars
WHAT="${one_artifact}" dump_artifact_config
declare WHAT_UPPERCASE="${one_artifact^^}"
declare WHAT_UPPERCASE_REPLACED="${WHAT_UPPERCASE//[-.]/_}"
all_wanted_artifact_names+=("${one_artifact}")
all_wanted_artifact_vars+=("${WHAT_UPPERCASE_REPLACED}")
eval "declare -r -g WANT_ARTIFACT_${WHAT_UPPERCASE_REPLACED}_INPUTS_ARRAY=\"${artifact_input_vars}\""
done
declare -r -g WANT_ARTIFACT_ALL_NAMES_ARRAY="${all_wanted_artifact_names[*]}"
declare -r -g WANT_ARTIFACT_ALL_ARRAY="${all_wanted_artifact_vars[*]}"
}