redhat/kernel.spec: add uki_addons to create UKI kernel cmdline addons

JIRA: https://issues.redhat.com/browse/RHEL-45159
Upstream Status: RHEL-Only

The folder redhat/uki_addons will contain all addons configs specifying the
UKI kernel cmdline addons to be created in the next build. An addon
config is simply a .addon plain text file, where any line
is taken as kernel cmdline, except for the ones starting with '#',
which will be automatically ignored.

redhat/scripts/uki_create_json.py will take care of parsing all configs and
folders in redhat/uki_addons and produce a json file 'uki_addons.json'
that is then added into the SRPM.
When building the RPM, kernel.spec will then call uki_create_addons.py
to parse the json and call 'ukify' for each addon matching the given
distro, arch and uki provided in input.

The output addon filename will be a concatenation of all folders in
redhat/uki_addons that are part of the addon config path.

The folder hierarchy inside of redhat/uki_addons is similar to
redhat/configs: $distro/$UKI_NAME/%arch.

It is also possible to add .sbat to all the generated addons, by
populating redhat/uki_addons/$distro/$UKI_NAME/%arch/sbat/sbat.conf.
Syntax is same as the addons config.

Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>

redhat/.gitignore

@@ -6,3 +6,4 @@ kabi/Module.kabi_*
 kabi/kabi-current
 kabi/kabi-rhel*
 kabi/kabi-rhel*/*
+scripts/uki_addons/uki_addons.json

redhat/Makefile

@@ -625,6 +625,7 @@ sources-rh: $(TARBALL) generate-testpatch-tmp setup-source dist-configs-check
 	@sed -e "s/%%SPECKVERSION%%/$(SPECKVERSION)/" \
 		-e "s/%%SPECKPATCHLEVEL%%/$(SPECKPATCHLEVEL)/" \
 		rpminspect.yaml > $(SOURCES)/rpminspect.yaml
+	@$(REDHAT)/scripts/uki_addons/uki_create_json.py $(REDHAT)/scripts/uki_addons/uki_addons.json
 	@cp cpupower.* \
 		keys/rhel*.x509 \
 		keys/nvidia*.x509 \
@@ -639,6 +640,8 @@ sources-rh: $(TARBALL) generate-testpatch-tmp setup-source dist-configs-check
 		mod-partner.list \
 		mod-kvm.list \
 		mod-sign.sh \
+		scripts/uki_addons/uki_create_addons.py \
+		scripts/uki_addons/uki_addons.json \
 		configs/flavors \
 		configs/generate_all_configs.sh \
 		configs/merge.pl \

redhat/kernel.spec.template

@@ -785,6 +785,8 @@ BuildRequires: lvm2
 BuildRequires: systemd-boot-unsigned
 # For systemd-pcrphase
 BuildRequires: systemd-udev >= 252-1
+# For UKI kernel cmdline addons
+BuildRequires: systemd-ukify
 # For TPM operations in UKI initramfs
 BuildRequires: tpm2-tools
 # For Azure CVM specific udev rules
@@ -929,6 +931,9 @@ Source105: nvidiagpuoot001.x509
 
 Source150: dracut-virt.conf
 
+Source151: uki_create_addons.py
+Source152: uki_addons.json
+
 Source200: check-kabi
 
 Source201: Module.kabi_aarch64
@@ -1504,6 +1509,11 @@ Provides: kernel-%{?1:%{1}-}uname-r = %{KVERREL}%{uname_suffix %{?1:%{1}}}\
 Requires: kernel%{?1:-%{1}}-modules-core-uname-r = %{KVERREL}%{uname_suffix %{?1:%{1}}}\
 Requires(pre): %{kernel_prereq}\
 Requires(pre): systemd >= 252-20\
+%package %{?1:%{1}-}uki-virt-addons\
+Summary: %{variant_summary} unified kernel image addons for virtual machines\
+Provides: installonlypkg(kernel)\
+Requires: kernel%{?1:-%{1}}-uki-virt = %{version}-%{release}\
+Requires(pre): systemd >= 252-20\
 %endif\
 %endif\
 %if "%{1}" == "rt" || "%{1}" == "rt-debug"\
@@ -1624,8 +1634,14 @@ input and output, etc.
 %description debug-uki-virt
 Prebuilt debug unified kernel image for virtual machines.
 
+%description debug-uki-virt-addons
+Prebuilt debug unified kernel image addons for virtual machines.
+
 %description uki-virt
 Prebuilt default unified kernel image for virtual machines.
+
+%description uki-virt-addons
+Prebuilt default unified kernel image addons for virtual machines.
 %endif
 
 %if %{with_ipaclones}
@@ -2466,6 +2482,10 @@ BuildKernel() {
                --kernel-cmdline 'console=tty0 console=ttyS0' \
     	   $KernelUnifiedImage
 
+        KernelAddonsDirOut="$KernelUnifiedImage.extra.d"
+        mkdir -p $KernelAddonsDirOut
+        python3 %{SOURCE151} %{SOURCE152} $KernelAddonsDirOut virt %{primary_target} %{_target_cpu}
+
 %if %{signkernel}
 
 %if 0%{?centos}
@@ -2482,6 +2502,12 @@ BuildKernel() {
         fi
         mv $KernelUnifiedImage.signed $KernelUnifiedImage
 
+      for addon in "$KernelAddonsDirOut"/*; do
+        %pesign -s -i $addon -o $addon.signed -a %{secureboot_ca_0} -c %{secureboot_key_0} -n %{pesign_name_0}
+        rm -f $addon
+        mv $addon.signed $addon
+      done
+
 # signkernel
 %endif
 
@@ -3680,6 +3706,9 @@ fi
 /lib/modules/%{KVERREL}%{?3:+%{3}}/modules.builtin*\
 /lib/modules/%{KVERREL}%{?3:+%{3}}/%{?-k:%{-k*}}%{!?-k:vmlinuz}-virt.efi\
 %ghost /%{image_install_path}/efi/EFI/Linux/%{?-k:%{-k*}}%{!?-k:*}-%{KVERREL}%{?3:+%{3}}.efi\
+%{expand:%%files %{?3:%{3}-}uki-virt-addons}\
+/lib/modules/%{KVERREL}%{?3:+%{3}}/%{?-k:%{-k*}}%{!?-k:vmlinuz}-virt.efi.extra.d/ \
+/lib/modules/%{KVERREL}%{?3:+%{3}}/%{?-k:%{-k*}}%{!?-k:vmlinuz}-virt.efi.extra.d/*.addon.efi\
 %endif\
 %endif\
 %if %{?3:1} %{!?3:0}\

redhat/scripts/uki_addons/uki_create_addons.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+#
+# This script inspects a given json proving a list of addons, and
+# creates an addon for each key/value pair matching the given uki, distro and
+# arch provided in input.
+#
+# Usage: python uki_create_addons.py input_json out_dir uki distro arch
+#
+# This tool requires the systemd-ukify and systemd-boot packages.
+#
+# Addon file
+#-----------
+# Each addon terminates with .addon
+# Each addon contains only two types of lines:
+# Lines beginning with '#' are description and thus ignored
+# All other lines are command line to be added.
+# The name of the end resulting addon is taken from the json hierarchy.
+# For example, and addon in json['virt']['rhel']['x86_64']['hello.addon'] will
+# result in an UKI addon file generated in out_dir called
+# hello-virt.rhel.x86_64.addon.efi
+#
+# The common key, present in any sub-dict in the provided json (except the leaf dict)
+# is used as place for default addons when the same addon is not defined deep
+# in the hierarchy. For example, if we define test.addon (text: 'test1\n') in
+# json['common']['test.addon'] = ['test1\n'] and another test.addon (text: test2) in
+# json['virt']['common']['test.addon'] = ['test2'], any other uki except virt
+# will have a test.addon.efi with text "test1", and virt will have a
+# test.addon.efi with "test2"
+#
+# sbat.conf
+#----------
+# This dict is containing the sbat string for *all* addons being created.
+# This dict is optional, but when used has to be put in a sub-dict with
+# { 'sbat' : { 'sbat.conf' : ['your text here'] }}
+# It follows the same syntax as the addon files, meaning '#' is comment and
+# the rest is taken as sbat string and feed to ukify.
+
+import os
+import sys
+import json
+import collections
+import subprocess
+
+
+UKIFY_PATH = '/usr/lib/systemd/ukify'
+
+def usage(err):
+    print(f'Usage: {os.path.basename(__file__)} input_json output_dir uki distro arch')
+    print(f'Error:{err}')
+    sys.exit(1)
+
+def check_clean_arguments(input_json, out_dir):
+    # Remove end '/'
+    if out_dir[-1:] == '/':
+        out_dir = out_dir[:-1]
+    if not os.path.isfile(input_json):
+        usage(f'input_json {input_json} is not a file, or does not exist!')
+    if not os.path.isdir(out_dir):
+        usage(f'out_dir_dir {out_dir} is not a dir, or does not exist!')
+    return out_dir
+
+UKICmdlineAddon = collections.namedtuple('UKICmdlineAddon', ['name', 'cmdline'])
+uki_addons_list = []
+uki_addons = {}
+addon_sbat_string = None
+
+def parse_lines(lines, rstrip=True):
+    cmdline = ''
+    for l in lines:
+        l = l.lstrip()
+        if not l:
+            continue
+        if l[0] == '#':
+            continue
+        # rstrip is used only for addons cmdline, not sbat.conf, as it replaces
+        # return lines with spaces.
+        if rstrip:
+            l = l.rstrip() + ' '
+        cmdline += l
+    if cmdline == '':
+        return ''
+    return cmdline
+
+def parse_all_addons(in_obj):
+    global addon_sbat_string
+
+    for el in in_obj.keys():
+        # addon found: copy it in our global dict uki_addons
+        if el.endswith('.addon'):
+            uki_addons[el] = in_obj[el]
+
+    if 'sbat' in in_obj and 'sbat.conf' in in_obj['sbat']:
+        # sbat.conf found: override sbat with the most specific one found
+        addon_sbat_string = parse_lines(in_obj['sbat']['sbat.conf'], rstrip=False)
+
+def recursively_find_addons(in_obj, folder_list):
+    # end of recursion, leaf directory. Search all addons here
+    if len(folder_list) == 0:
+        parse_all_addons(in_obj)
+        return
+
+    # first, check for common folder
+    if 'common' in in_obj:
+        parse_all_addons(in_obj['common'])
+
+    # second, check if there is a match with the searched folder
+    if folder_list[0] in in_obj:
+        folder_next = in_obj[folder_list[0]]
+        folder_list = folder_list[1:]
+        recursively_find_addons(folder_next, folder_list)
+
+def parse_in_json(in_json, uki_name, distro, arch):
+    with open(in_json, 'r') as f:
+        in_obj = json.load(f)
+    recursively_find_addons(in_obj, [uki_name, distro, arch])
+
+    for addon_name, cmdline in uki_addons.items():
+        addon_name = addon_name.replace(".addon","")
+        addon_full_name = f'{addon_name}-{uki_name}.{distro}.{arch}.addon.efi'
+        cmdline = parse_lines(cmdline).rstrip()
+        if cmdline:
+            uki_addons_list.append(UKICmdlineAddon(addon_full_name, cmdline))
+
+def create_addons(out_dir):
+    for uki_addon in uki_addons_list:
+        out_path = os.path.join(out_dir, uki_addon.name)
+        cmd = [
+            f'{UKIFY_PATH}', 'build',
+            f'--cmdline="{uki_addon.cmdline}"',
+            f'--output={out_path}']
+        if addon_sbat_string:
+            cmd.append('--sbat="' + addon_sbat_string.rstrip() +'"')
+
+        subprocess.check_call(cmd, text=True)
+
+if __name__ == "__main__":
+    argc = len(sys.argv) - 1
+    if argc != 5:
+        usage('too few or too many parameters!')
+
+    input_json = sys.argv[1]
+    out_dir = sys.argv[2]
+    uki_name = sys.argv[3]
+    distro = sys.argv[4]
+    arch = sys.argv[5]
+
+    out_dir = check_clean_arguments(input_json, out_dir)
+    parse_in_json(input_json, uki_name, distro, arch)
+    create_addons(out_dir)
+
+

redhat/scripts/uki_addons/uki_create_json.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+#
+# This script recursively inspects the 'redhat/uki_addons' directory,
+# and creates a json file containing name and description of each addon found.
+#
+# Usage: python uki_create_json.py output_file
+#
+# Addon file
+#-----------
+# The addon files are contained into the 'redhat/uki_addons' folder.
+# Each addon terminates with .addon.
+# Each addon contains only two types of lines:
+# Lines beginning with '#' are description and thus ignored
+# All other lines are command line to be added.
+# This script just parses the folder structure and creates a json reflecting
+# all addons and their line found.
+# For example, if we define test.addon (text: 'test1\n') in
+# redhat/uki_addons/virt/rhel/x86_64, the resulting output_file will contain
+# { 'virt' : { 'rhel' : { 'x86_64' : { 'test.addon' : ['test1\n'] }}}}
+#
+# The name of the end resulting addon is taken from the folder hierarchy, but this
+# is handled by uki_create_addons.py when building the rpm. This script only
+# prepares the json file to be added in the srpm. For more information about
+# the folder hierarchy, what the 'common' and 'sbat' folder are, look at
+# uki_create_addons.py.
+#
+# The common folder, present in any folder under redhat/uki_addons
+# (except the leaf folders) is used as place for default addons when the same
+# addon is not defined deep in the hierarchy.
+#
+# How to extend the script and kernel.spec with a new arch or uki or distro
+#--------------------------------------------------------------------------
+# A new distro has to be added by creating the folder in redhat/uki_addons.
+# See uki_create_addons.py to how the directory hierarchy in redhat/uki_addons
+# is expected to be.
+# After that, if the distro is a different arch from the one already supported,
+# one needs to extend the %define with_efiuki in kernel.spec.template.
+# If a new UKI has to be created with a different name from the existing ones,
+# the logic to create the addons and call this script has to be implemented too
+# in kernel.spec.template. As an example, see how the 'virt' UKI addons are
+# created.
+
+import os
+import sys
+import json
+import subprocess
+
+def usage(err):
+    print(f'Usage: {os.path.basename(__file__)} dest_file')
+    print(f'Error:{err}')
+    sys.exit(1)
+
+def find_addons():
+    cmd = ['/usr/bin/find', 'uki_addons', "(", '-name', '*.addon', '-o', '-name', 'sbat.conf', ")"]
+    proc_out = subprocess.run(cmd, check=True, capture_output=True, text=True)
+    if proc_out.returncode == 0:
+        return proc_out.stdout
+    return None
+
+def add_keys_to_obj(ret_data, keys, value):
+    if len(keys) == 0:
+        return
+    key = keys[0]
+    val = {}
+    if len(keys) == 1:
+        val = value
+    if not key in ret_data:
+        ret_data[key] = val
+    ret_data = ret_data[key]
+    add_keys_to_obj(ret_data, keys[1:], value)
+
+def create_json(addons):
+    obj = {}
+    for el in addons:
+        print(f'Processing {el} ...')
+        with open(el, 'r') as f:
+            lines = f.readlines()
+        dirs, name = os.path.split(el)
+        keys = dirs.split('/')
+        if keys[0] == 'uki_addons':
+            keys = keys[1:]
+        keys.append(name)
+        add_keys_to_obj(obj, keys, lines)
+        print(f'Processing {el} completed')
+    return obj
+
+def write_json(obj, dest_file):
+    with open(dest_file, 'w') as f:
+        json.dump(obj , f, indent=4)
+    print(f'Processed addons files are in {dest_file}')
+
+if __name__ == "__main__":
+    argc = len(sys.argv) - 1
+    if argc != 1:
+        usage('too few or too many parameters!')
+    dest = sys.argv[1]
+
+    output = find_addons()
+    if output is None:
+        usage('error finding the addons')
+    addons_list = output.split()
+    obj = create_json(addons_list)
+    write_json(obj, dest)
+