qiurui/Centos-kernel-stream-9
1
0
Fork 0
mirror of https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9.git synced 2026-09-09 00:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

x86/entry: Optimize common_interrupt_return()

Browse Source 
JIRA: https://issues.redhat.com/browse/RHEL-25415

commit c516213726fb572700cce4a5909aa8d82b77192a
Author: Peter Zijlstra <peterz@infradead.org>
Date:   Mon Nov 20 15:33:45 2023 +0100

    x86/entry: Optimize common_interrupt_return()

    The code in common_interrupt_return() does a bunch of unconditional
    work that is really only needed on PTI kernels. Specifically it
    unconditionally copies the IRET frame back onto the entry stack,
    swizzles onto the entry stack and does IRET from there.

    However, without PTI we can simply IRET from whatever stack we're on.

      ivb-ep, mitigations=off, gettid-1m:

      PRE:
           140,118,538      cycles:k                                                      ( +-  0.01% )
           236,692,878      instructions:k            #    1.69  insn per cycle           ( +-  0.00% )

      POST:
           140,026,608      cycles:k                                                      ( +-  0.01% )
           236,696,176      instructions:k            #    1.69  insn per cycle           ( +-  0.00% )

    (this is with --repeat 100 and the run-to-run variance is bigger than
    the difference shown)

    Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
    Signed-off-by: Ingo Molnar <mingo@kernel.org>
    Cc: Linus Torvalds <torvalds@linux-foundation.org>
    Link: https://lore.kernel.org/r/20231120143626.638107480@infradead.org

Signed-off-by: Prarit Bhargava <prarit@redhat.com>
This commit is contained in:
Prarit Bhargava
2024-03-20 09:43:35 -04:00
parent 198dfcf478
commit 06c04a3a47
2 changed files with 24 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
arch/x86/entry/calling.h
+9 -3
View File
@@ -175,8 +175,7 @@ For 32-bit we have the following conventions - kernel is built with
#define THIS_CPU_user_pcid_flush_mask \
PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req
mov %cr3, \scratch_reg
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
@@ -206,13 +205,20 @@ For 32-bit we have the following conventions - kernel is built with
/* Flip the PGD to the user version */
orq $(PTI_USER_PGTABLE_MASK), \scratch_reg
mov \scratch_reg, %cr3
.endm
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2
.Lend_\@:
.endm
.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
pushq %rax
SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax
SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax
popq %rax
.Lend_\@:
.endm
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
arch/x86/entry/entry_64.S
+15 -2
View File
@@ -572,7 +572,18 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
#ifdef CONFIG_XEN_PV
ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
#endif
#ifdef CONFIG_PAGE_TABLE_ISOLATION
ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
#endif
STACKLEAK_ERASE
POP_REGS
add $8, %rsp /* orig_ax */
swapgs
jmp .Lnative_iret
#ifdef CONFIG_PAGE_TABLE_ISOLATION
.Lpti_restore_regs_and_return_to_usermode:
POP_REGS pop_rdi=0
/*
@@ -599,13 +610,15 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
*/
STACKLEAK_ERASE_NOCLOBBER
SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
push %rax
SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
pop %rax
/* Restore RDI. */
popq %rdi
swapgs
jmp .Lnative_iret
#endif
SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
#ifdef CONFIG_DEBUG_ENTRY