mirror of
https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-10.git
synced 2026-09-09 00:07:04 +08:00
Phil Auld
5f86bda714
JIRA: https://issues.redhat.com/browse/RHEL-124642 Conflicts: Minor context difference in include/linux/sched.h commit 378b7708194fff77c9020392067329931c3fcc04 Author: Menglong Dong <menglong8.dong@gmail.com> Date: Wed Sep 17 14:09:15 2025 +0800 sched: Make migrate_{en,dis}able() inline For now, migrate_enable and migrate_disable are global, which makes them become hotspots in some case. Take BPF for example, the function calling to migrate_enable and migrate_disable in BPF trampoline can introduce significant overhead, and following is the 'perf top' of FENTRY's benchmark (./tools/testing/selftests/bpf/bench trig-fentry): 54.63% bpf_prog_2dcccf652aac1793_bench_trigger_fentry [k] bpf_prog_2dcccf652aac1793_bench_trigger_fentry 10.43% [kernel] [k] migrate_enable 10.07% bpf_trampoline_6442517037 [k] bpf_trampoline_6442517037 8.06% [kernel] [k] __bpf_prog_exit_recur 4.11% libc.so.6 [.] syscall 2.15% [kernel] [k] entry_SYSCALL_64 1.48% [kernel] [k] memchr_inv 1.32% [kernel] [k] fput 1.16% [kernel] [k] _copy_to_user 0.73% [kernel] [k] bpf_prog_test_run_raw_tp So in this commit, we make migrate_enable/migrate_disable inline to obtain better performance. The struct rq is defined internally in kernel/sched/sched.h, and the field "nr_pinned" is accessed in migrate_enable/migrate_disable, which makes it hard to make them inline. Alexei Starovoitov suggests to generate the offset of "nr_pinned" in [1], so we can define the migrate_enable/migrate_disable in include/linux/sched.h and access "this_rq()->nr_pinned" with "(void *)this_rq() + RQ_nr_pinned". The offset of "nr_pinned" is generated in include/generated/rq-offsets.h by kernel/sched/rq-offsets.c. Generally speaking, we move the definition of migrate_enable and migrate_disable to include/linux/sched.h from kernel/sched/core.c. The calling to __set_cpus_allowed_ptr() is leaved in ___migrate_enable(). The "struct rq" is not available in include/linux/sched.h, so we can't access the "runqueues" with this_cpu_ptr(), as the compilation will fail in this_cpu_ptr() -> raw_cpu_ptr() -> __verify_pcpu_ptr(): typeof((ptr) + 0) So we introduce the this_rq_raw() and access the runqueues with arch_raw_cpu_ptr/PERCPU_PTR directly. The variable "runqueues" is not visible in the kernel modules, and export it is not a good idea. As Peter Zijlstra advised in [2], we define and export migrate_enable/migrate_disable in kernel/sched/core.c too, and use them for the modules. Before this patch, the performance of BPF FENTRY is: fentry : 113.030 ± 0.149M/s fentry : 112.501 ± 0.187M/s fentry : 112.828 ± 0.267M/s fentry : 115.287 ± 0.241M/s After this patch, the performance of BPF FENTRY increases to: fentry : 143.644 ± 0.670M/s fentry : 149.764 ± 0.362M/s fentry : 149.642 ± 0.156M/s fentry : 145.263 ± 0.221M/s Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/bpf/CAADnVQ+5sEDKHdsJY5ZsfGDO_1SEhhQWHrt2SMBG5SYyQ+jt7w@mail.gmail.com/ [1] Link: https://lore.kernel.org/all/20250819123214.GH4067720@noisy.programming.kicks-ass.net/ [2] Signed-off-by: Phil Auld <pauld@redhat.com>
111 lines
2.8 KiB
Makefile
111 lines
2.8 KiB
Makefile
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Kbuild for top-level directory of the kernel
|
|
|
|
# Prepare global headers and check sanity before descending into sub-directories
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Generate bounds.h
|
|
|
|
bounds-file := include/generated/bounds.h
|
|
|
|
targets := kernel/bounds.s
|
|
|
|
$(bounds-file): kernel/bounds.s FORCE
|
|
$(call filechk,offsets,__LINUX_BOUNDS_H__)
|
|
|
|
# Generate timeconst.h
|
|
|
|
timeconst-file := include/generated/timeconst.h
|
|
|
|
filechk_gentimeconst = echo $(CONFIG_HZ) | bc -q $<
|
|
|
|
$(timeconst-file): kernel/time/timeconst.bc FORCE
|
|
$(call filechk,gentimeconst)
|
|
|
|
# Generate asm-offsets.h
|
|
|
|
offsets-file := include/generated/asm-offsets.h
|
|
|
|
targets += arch/$(SRCARCH)/kernel/asm-offsets.s
|
|
|
|
arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file)
|
|
|
|
$(offsets-file): arch/$(SRCARCH)/kernel/asm-offsets.s FORCE
|
|
$(call filechk,offsets,__ASM_OFFSETS_H__)
|
|
|
|
# Generate rq-offsets.h
|
|
|
|
rq-offsets-file := include/generated/rq-offsets.h
|
|
|
|
targets += kernel/sched/rq-offsets.s
|
|
|
|
kernel/sched/rq-offsets.s: $(offsets-file)
|
|
|
|
$(rq-offsets-file): kernel/sched/rq-offsets.s FORCE
|
|
$(call filechk,offsets,__RQ_OFFSETS_H__)
|
|
|
|
# Check for missing system calls
|
|
|
|
quiet_cmd_syscalls = CALL $<
|
|
cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags)
|
|
|
|
PHONY += missing-syscalls
|
|
missing-syscalls: scripts/checksyscalls.sh $(rq-offsets-file)
|
|
$(call cmd,syscalls)
|
|
|
|
# Check the manual modification of atomic headers
|
|
|
|
quiet_cmd_check_sha1 = CHKSHA1 $<
|
|
cmd_check_sha1 = \
|
|
if ! command -v sha1sum >/dev/null; then \
|
|
echo "warning: cannot check the header due to sha1sum missing"; \
|
|
exit 0; \
|
|
fi; \
|
|
if [ "$$(sed -n '$$s:// ::p' $<)" != \
|
|
"$$(sed '$$d' $< | sha1sum | sed 's/ .*//')" ]; then \
|
|
echo "error: $< has been modified." >&2; \
|
|
exit 1; \
|
|
fi; \
|
|
touch $@
|
|
|
|
atomic-checks += $(addprefix $(obj)/.checked-, \
|
|
atomic-arch-fallback.h \
|
|
atomic-instrumented.h \
|
|
atomic-long.h)
|
|
|
|
targets += $(atomic-checks)
|
|
$(atomic-checks): $(obj)/.checked-%: include/linux/atomic/% FORCE
|
|
$(call if_changed,check_sha1)
|
|
|
|
# A phony target that depends on all the preparation targets
|
|
|
|
PHONY += prepare
|
|
prepare: $(offsets-file) missing-syscalls $(atomic-checks)
|
|
@:
|
|
|
|
# Ordinary directory descending
|
|
# ---------------------------------------------------------------------------
|
|
|
|
obj-y += init/
|
|
obj-y += usr/
|
|
obj-y += arch/$(SRCARCH)/
|
|
obj-y += $(ARCH_CORE)
|
|
obj-y += kernel/
|
|
obj-y += certs/
|
|
obj-y += mm/
|
|
obj-y += fs/
|
|
obj-y += ipc/
|
|
obj-y += security/
|
|
obj-y += crypto/
|
|
obj-$(CONFIG_BLOCK) += block/
|
|
obj-$(CONFIG_IO_URING) += io_uring/
|
|
obj-$(CONFIG_RUST) += rust/
|
|
obj-y += $(ARCH_LIB)
|
|
obj-y += drivers/
|
|
obj-y += sound/
|
|
obj-$(CONFIG_SAMPLES) += samples/
|
|
obj-$(CONFIG_NET) += net/
|
|
obj-y += virt/
|
|
obj-y += $(ARCH_DRIVERS)
|