Merge: KVM: Host kernel panics with "unexpected #NM exception" at restore_fpregs_from_fpstate

MR: https://gitlab.com/redhat/rhel/src/kernel/rhel-10/-/merge_requests/797

Until now, fpregs->xfd acted as both the guest value and the value that
the host used when executing XSAVES and XRSTORS.  This is wrong; the
data in the guest's FPU might not be initialized even if a bit is
set in XFD, and when that happens XRSTORing the guest FPU will fail
with a #NM exception.

Instead, store the value of XFD together with XFD_ERR in struct
fpu_guest, and synchronize it in fpregs_ensure_guest_state().

JIRA: https://issues.redhat.com/browse/RHEL-148618

Tested: kvm unit tests and selftests on Intel and AMD machine.

Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>

Approved-by: Vitaly Kuznetsov <vkuznets@redhat.com>
Approved-by: Steve Best <sbest@redhat.com>
Approved-by: CKI KWF Bot <cki-ci-bot+kwf-gitlab-com@redhat.com>

Merged-by: CKI GitLab Kmaint Pipeline Bot <26919896-cki-kmaint-pipeline-bot@users.noreply.gitlab.com>
This commit is contained in:
CKI KWF Bot
2026-03-06 11:09:59 +00:00
12 changed files with 195 additions and 87 deletions
+5
View File
@@ -35,6 +35,11 @@
#define MC_VECTOR 18
#define XM_VECTOR 19
#define VE_VECTOR 20
#define CP_VECTOR 21
#define HV_VECTOR 28
#define VC_VECTOR 29
#define SX_VECTOR 30
/* Select x86 specific features in <linux/kvm.h> */
#define __KVM_HAVE_PIT
+29 -3
View File
@@ -294,10 +294,29 @@ EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features);
#ifdef CONFIG_X86_64
void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd)
{
struct fpstate *fpstate = guest_fpu->fpstate;
fpregs_lock();
guest_fpu->fpstate->xfd = xfd;
if (guest_fpu->fpstate->in_use)
xfd_update_state(guest_fpu->fpstate);
/*
* KVM's guest ABI is that setting XFD[i]=1 *can* immediately revert the
* save state to its initial configuration. Likewise, KVM_GET_XSAVE does
* the same as XSAVE and returns XSTATE_BV[i]=0 whenever XFD[i]=1.
*
* If the guest's FPU state is in hardware, just update XFD: the XSAVE
* in fpu_swap_kvm_fpstate will clear XSTATE_BV[i] whenever XFD[i]=1.
*
* If however the guest's FPU state is NOT resident in hardware, clear
* disabled components in XSTATE_BV now, or a subsequent XRSTOR will
* attempt to load disabled components and generate #NM _in the host_.
*/
if (xfd && test_thread_flag(TIF_NEED_FPU_LOAD))
fpstate->regs.xsave.header.xfeatures &= ~xfd;
fpstate->xfd = xfd;
if (fpstate->in_use)
xfd_update_state(fpstate);
fpregs_unlock();
}
EXPORT_SYMBOL_GPL(fpu_update_guest_xfd);
@@ -405,6 +424,13 @@ int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf,
if (ustate->xsave.header.xfeatures & ~xcr0)
return -EINVAL;
/*
* Disabled features must be in their initial state, otherwise XRSTOR
* causes an exception.
*/
if (WARN_ON_ONCE(ustate->xsave.header.xfeatures & kstate->xfd))
return -EINVAL;
/*
* Nullify @vpkru to preserve its current value if PKRU's bit isn't set
* in the header. KVM's odd ABI is to leave PKRU untouched in this
+3 -2
View File
@@ -461,8 +461,9 @@ TRACE_EVENT(kvm_inj_virq,
#define kvm_trace_sym_exc \
EXS(DE), EXS(DB), EXS(BP), EXS(OF), EXS(BR), EXS(UD), EXS(NM), \
EXS(DF), EXS(TS), EXS(NP), EXS(SS), EXS(GP), EXS(PF), \
EXS(MF), EXS(AC), EXS(MC)
EXS(DF), EXS(TS), EXS(NP), EXS(SS), EXS(GP), EXS(PF), EXS(MF), \
EXS(AC), EXS(MC), EXS(XM), EXS(VE), EXS(CP), \
EXS(HV), EXS(VC), EXS(SX)
/*
* Tracepoint for kvm interrupt injection:
+9
View File
@@ -5676,9 +5676,18 @@ static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
struct kvm_xsave *guest_xsave)
{
union fpregs_state *xstate = (union fpregs_state *)guest_xsave->region;
if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
/*
* For backwards compatibility, do not expect disabled features to be in
* their initial state. XSTATE_BV[i] must still be cleared whenever
* XFD[i]=1, or XRSTOR would cause a #NM.
*/
xstate->xsave.header.xfeatures &= ~vcpu->arch.guest_fpu.fpstate->xfd;
return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
guest_xsave->region,
kvm_caps.supported_xcr0,
@@ -34,6 +34,8 @@ extern uint64_t guest_tsc_khz;
#define NMI_VECTOR 0x02
const char *ex_str(int vector);
#define X86_EFLAGS_FIXED (1u << 1)
#define X86_CR4_VME (1ul << 0)
@@ -1184,6 +1186,12 @@ struct idt_entry {
void vm_install_exception_handler(struct kvm_vm *vm, int vector,
void (*handler)(struct ex_regs *));
/*
* Exception fixup morphs #DE to an arbitrary magic vector so that '0' can be
* used to signal "no expcetion".
*/
#define KVM_MAGIC_DE_VECTOR 0xff
/* If a toddler were to say "abracadabra". */
#define KVM_EXCEPTION_MAGIC 0xabacadabaULL
@@ -24,6 +24,39 @@ bool host_cpu_is_intel;
bool is_forced_emulation_enabled;
uint64_t guest_tsc_khz;
const char *ex_str(int vector)
{
switch (vector) {
#define VEC_STR(v) case v##_VECTOR: return "#" #v
case DE_VECTOR: return "no exception";
case KVM_MAGIC_DE_VECTOR: return "#DE";
VEC_STR(DB);
VEC_STR(NMI);
VEC_STR(BP);
VEC_STR(OF);
VEC_STR(BR);
VEC_STR(UD);
VEC_STR(NM);
VEC_STR(DF);
VEC_STR(TS);
VEC_STR(NP);
VEC_STR(SS);
VEC_STR(GP);
VEC_STR(PF);
VEC_STR(MF);
VEC_STR(AC);
VEC_STR(MC);
VEC_STR(XM);
VEC_STR(VE);
VEC_STR(CP);
VEC_STR(HV);
VEC_STR(VC);
VEC_STR(SX);
default: return "#??";
#undef VEC_STR
}
}
static void regs_dump(FILE *stream, struct kvm_regs *regs, uint8_t indent)
{
fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
@@ -558,7 +591,7 @@ static bool kvm_fixup_exception(struct ex_regs *regs)
return false;
if (regs->vector == DE_VECTOR)
return false;
regs->vector = KVM_MAGIC_DE_VECTOR;
regs->rip = regs->r11;
regs->r9 = regs->vector;
+85 -59
View File
@@ -69,6 +69,12 @@ static inline void __tileloadd(void *tile)
: : "a"(tile), "d"(0));
}
static inline int tileloadd_safe(void *tile)
{
return kvm_asm_safe(".byte 0xc4,0xe2,0x7b,0x4b,0x04,0x10",
"a"(tile), "d"(0));
}
static inline void __tilerelease(void)
{
asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0" ::);
@@ -124,27 +130,52 @@ static void set_tilecfg(struct tile_config *cfg)
}
}
enum {
/* Retrieve TMM0 from guest, stash it for TEST_RESTORE_TILEDATA */
TEST_SAVE_TILEDATA = 1,
/* Check TMM0 against tiledata */
TEST_COMPARE_TILEDATA = 2,
/* Restore TMM0 from earlier save */
TEST_RESTORE_TILEDATA = 4,
/* Full VM save/restore */
TEST_SAVE_RESTORE = 8,
};
static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
struct tile_data *tiledata,
struct xstate *xstate)
{
int vector;
GUEST_ASSERT(this_cpu_has(X86_FEATURE_XSAVE) &&
this_cpu_has(X86_FEATURE_OSXSAVE));
check_xtile_info();
GUEST_SYNC(1);
GUEST_SYNC(TEST_SAVE_RESTORE);
/* xfd=0, enable amx */
wrmsr(MSR_IA32_XFD, 0);
GUEST_SYNC(2);
GUEST_SYNC(TEST_SAVE_RESTORE);
GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == 0);
set_tilecfg(amx_cfg);
__ldtilecfg(amx_cfg);
GUEST_SYNC(3);
GUEST_SYNC(TEST_SAVE_RESTORE);
/* Check save/restore when trap to userspace */
__tileloadd(tiledata);
GUEST_SYNC(4);
GUEST_SYNC(TEST_SAVE_TILEDATA | TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE);
/* xfd=0x40000, disable amx tiledata */
wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA);
/* host tries setting tiledata while guest XFD is set */
GUEST_SYNC(TEST_RESTORE_TILEDATA);
GUEST_SYNC(TEST_SAVE_RESTORE);
wrmsr(MSR_IA32_XFD, 0);
__tilerelease();
GUEST_SYNC(5);
GUEST_SYNC(TEST_SAVE_RESTORE);
/*
* After XSAVEC, XTILEDATA is cleared in the xstate_bv but is set in
* the xcomp_bv.
@@ -154,6 +185,8 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA));
GUEST_ASSERT(xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA);
/* #NM test */
/* xfd=0x40000, disable amx tiledata */
wrmsr(MSR_IA32_XFD, XFEATURE_MASK_XTILE_DATA);
@@ -166,32 +199,33 @@ static void __attribute__((__flatten__)) guest_code(struct tile_config *amx_cfg,
GUEST_ASSERT(!(xstate->header.xstate_bv & XFEATURE_MASK_XTILE_DATA));
GUEST_ASSERT((xstate->header.xcomp_bv & XFEATURE_MASK_XTILE_DATA));
GUEST_SYNC(6);
GUEST_SYNC(TEST_SAVE_RESTORE);
GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
set_tilecfg(amx_cfg);
__ldtilecfg(amx_cfg);
/* Trigger #NM exception */
__tileloadd(tiledata);
GUEST_SYNC(10);
vector = tileloadd_safe(tiledata);
__GUEST_ASSERT(vector == NM_VECTOR,
"Wanted #NM on tileloadd with XFD[18]=1, got %s",
ex_str(vector));
GUEST_DONE();
}
void guest_nm_handler(struct ex_regs *regs)
{
/* Check if #NM is triggered by XFEATURE_MASK_XTILE_DATA */
GUEST_SYNC(7);
GUEST_ASSERT(!(get_cr0() & X86_CR0_TS));
GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA);
GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
GUEST_SYNC(8);
GUEST_SYNC(TEST_SAVE_RESTORE);
GUEST_ASSERT(rdmsr(MSR_IA32_XFD_ERR) == XFEATURE_MASK_XTILE_DATA);
GUEST_ASSERT(rdmsr(MSR_IA32_XFD) == XFEATURE_MASK_XTILE_DATA);
/* Clear xfd_err */
wrmsr(MSR_IA32_XFD_ERR, 0);
/* xfd=0, enable amx */
wrmsr(MSR_IA32_XFD, 0);
GUEST_SYNC(9);
GUEST_SYNC(TEST_SAVE_RESTORE);
__tileloadd(tiledata);
GUEST_SYNC(TEST_COMPARE_TILEDATA | TEST_SAVE_RESTORE);
GUEST_DONE();
}
int main(int argc, char *argv[])
@@ -200,10 +234,10 @@ int main(int argc, char *argv[])
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
struct kvm_x86_state *state;
struct kvm_x86_state *tile_state = NULL;
int xsave_restore_size;
vm_vaddr_t amx_cfg, tiledata, xstate;
struct ucall uc;
u32 amx_offset;
int ret;
/*
@@ -228,9 +262,6 @@ int main(int argc, char *argv[])
vcpu_regs_get(vcpu, &regs1);
/* Register #NM handler */
vm_install_exception_handler(vm, NM_VECTOR, guest_nm_handler);
/* amx cfg for guest_code */
amx_cfg = vm_vaddr_alloc_page(vm);
memset(addr_gva2hva(vm, amx_cfg), 0x0, getpagesize());
@@ -244,6 +275,7 @@ int main(int argc, char *argv[])
memset(addr_gva2hva(vm, xstate), 0, PAGE_SIZE * DIV_ROUND_UP(XSAVE_SIZE, PAGE_SIZE));
vcpu_args_set(vcpu, 3, amx_cfg, tiledata, xstate);
int iter = 0;
for (;;) {
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
@@ -253,37 +285,47 @@ int main(int argc, char *argv[])
REPORT_GUEST_ASSERT(uc);
/* NOT REACHED */
case UCALL_SYNC:
switch (uc.args[1]) {
case 1:
case 2:
case 3:
case 5:
case 6:
case 7:
case 8:
fprintf(stderr, "GUEST_SYNC(%ld)\n", uc.args[1]);
break;
case 4:
case 10:
fprintf(stderr,
"GUEST_SYNC(%ld), check save/restore status\n", uc.args[1]);
++iter;
if (uc.args[1] & TEST_SAVE_TILEDATA) {
fprintf(stderr, "GUEST_SYNC #%d, save tiledata\n", iter);
tile_state = vcpu_save_state(vcpu);
}
if (uc.args[1] & TEST_COMPARE_TILEDATA) {
fprintf(stderr, "GUEST_SYNC #%d, check TMM0 contents\n", iter);
/* Compacted mode, get amx offset by xsave area
* size subtract 8K amx size.
*/
amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
state = vcpu_save_state(vcpu);
void *amx_start = (void *)state->xsave + amx_offset;
u32 amx_offset = xsave_restore_size - NUM_TILES*TILE_SIZE;
void *amx_start = (void *)tile_state->xsave + amx_offset;
void *tiles_data = (void *)addr_gva2hva(vm, tiledata);
/* Only check TMM0 register, 1 tile */
ret = memcmp(amx_start, tiles_data, TILE_SIZE);
TEST_ASSERT(ret == 0, "memcmp failed, ret=%d", ret);
}
if (uc.args[1] & TEST_RESTORE_TILEDATA) {
fprintf(stderr, "GUEST_SYNC #%d, before KVM_SET_XSAVE\n", iter);
vcpu_xsave_set(vcpu, tile_state->xsave);
fprintf(stderr, "GUEST_SYNC #%d, after KVM_SET_XSAVE\n", iter);
}
if (uc.args[1] & TEST_SAVE_RESTORE) {
fprintf(stderr, "GUEST_SYNC #%d, save/restore VM state\n", iter);
state = vcpu_save_state(vcpu);
memset(&regs1, 0, sizeof(regs1));
vcpu_regs_get(vcpu, &regs1);
kvm_vm_release(vm);
/* Restore state in a new VM. */
vcpu = vm_recreate_with_one_vcpu(vm);
vcpu_load_state(vcpu, state);
kvm_x86_state_cleanup(state);
break;
case 9:
fprintf(stderr,
"GUEST_SYNC(%ld), #NM exception and enable amx\n", uc.args[1]);
break;
memset(&regs2, 0, sizeof(regs2));
vcpu_regs_get(vcpu, &regs2);
TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
"Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
(ulong) regs2.rdi, (ulong) regs2.rsi);
}
break;
case UCALL_DONE:
@@ -293,22 +335,6 @@ int main(int argc, char *argv[])
TEST_FAIL("Unknown ucall %lu", uc.cmd);
}
state = vcpu_save_state(vcpu);
memset(&regs1, 0, sizeof(regs1));
vcpu_regs_get(vcpu, &regs1);
kvm_vm_release(vm);
/* Restore state in a new VM. */
vcpu = vm_recreate_with_one_vcpu(vm);
vcpu_load_state(vcpu, state);
kvm_x86_state_cleanup(state);
memset(&regs2, 0, sizeof(regs2));
vcpu_regs_get(vcpu, &regs2);
TEST_ASSERT(!memcmp(&regs1, &regs2, sizeof(regs2)),
"Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx",
(ulong) regs2.rdi, (ulong) regs2.rsi);
}
done:
kvm_vm_free(vm);
@@ -54,12 +54,12 @@ static void guest_msr(struct msr_data *msr)
if (msr->fault_expected)
__GUEST_ASSERT(vector == GP_VECTOR,
"Expected #GP on %sMSR(0x%x), got vector '0x%x'",
msr->write ? "WR" : "RD", msr->idx, vector);
"Expected #GP on %sMSR(0x%x), got %s",
msr->write ? "WR" : "RD", msr->idx, ex_str(vector));
else
__GUEST_ASSERT(!vector,
"Expected success on %sMSR(0x%x), got vector '0x%x'",
msr->write ? "WR" : "RD", msr->idx, vector);
"Expected success on %sMSR(0x%x), got %s",
msr->write ? "WR" : "RD", msr->idx, ex_str(vector));
if (vector || is_write_only_msr(msr->idx))
goto done;
@@ -102,12 +102,12 @@ static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall)
vector = __hyperv_hypercall(hcall->control, input, output, &res);
if (hcall->ud_expected) {
__GUEST_ASSERT(vector == UD_VECTOR,
"Expected #UD for control '%lu', got vector '0x%x'",
hcall->control, vector);
"Expected #UD for control '%lu', got %s",
hcall->control, ex_str(vector));
} else {
__GUEST_ASSERT(!vector,
"Expected no exception for control '%lu', got vector '0x%x'",
hcall->control, vector);
"Expected no exception for control '%lu', got %s",
hcall->control, ex_str(vector));
GUEST_ASSERT_EQ(res, hcall->expect);
}
@@ -30,12 +30,12 @@ do { \
\
if (fault_wanted) \
__GUEST_ASSERT((vector) == UD_VECTOR, \
"Expected #UD on " insn " for testcase '0x%x', got '0x%x'", \
testcase, vector); \
"Expected #UD on " insn " for testcase '0x%x', got %s", \
testcase, ex_str(vector)); \
else \
__GUEST_ASSERT(!(vector), \
"Expected success on " insn " for testcase '0x%x', got '0x%x'", \
testcase, vector); \
"Expected success on " insn " for testcase '0x%x', got %s", \
testcase, ex_str(vector)); \
} while (0)
static void guest_monitor_wait(void *arg)
@@ -365,8 +365,8 @@ static void test_arch_events(uint8_t pmu_version, uint64_t perf_capabilities,
#define GUEST_ASSERT_PMC_MSR_ACCESS(insn, msr, expect_gp, vector) \
__GUEST_ASSERT(expect_gp ? vector == GP_VECTOR : !vector, \
"Expected %s on " #insn "(0x%x), got vector %u", \
expect_gp ? "#GP" : "no fault", msr, vector) \
"Expected %s on " #insn "(0x%x), got %s", \
expect_gp ? "#GP" : "no fault", msr, ex_str(vector)) \
#define GUEST_ASSERT_PMC_VALUE(insn, msr, val, expected) \
__GUEST_ASSERT(val == expected, \
@@ -57,8 +57,8 @@ static void guest_test_perf_capabilities_gp(uint64_t val)
uint8_t vector = wrmsr_safe(MSR_IA32_PERF_CAPABILITIES, val);
__GUEST_ASSERT(vector == GP_VECTOR,
"Expected #GP for value '0x%lx', got vector '0x%x'",
val, vector);
"Expected #GP for value '0x%lx', got %s",
val, ex_str(vector));
}
static void guest_code(uint64_t current_val)
@@ -81,13 +81,13 @@ static void guest_code(void)
vector = xsetbv_safe(0, XFEATURE_MASK_FP);
__GUEST_ASSERT(!vector,
"Expected success on XSETBV(FP), got vector '0x%x'",
vector);
"Expected success on XSETBV(FP), got %s",
ex_str(vector));
vector = xsetbv_safe(0, supported_xcr0);
__GUEST_ASSERT(!vector,
"Expected success on XSETBV(0x%lx), got vector '0x%x'",
supported_xcr0, vector);
"Expected success on XSETBV(0x%lx), got %s",
supported_xcr0, ex_str(vector));
for (i = 0; i < 64; i++) {
if (supported_xcr0 & BIT_ULL(i))
@@ -95,8 +95,8 @@ static void guest_code(void)
vector = xsetbv_safe(0, supported_xcr0 | BIT_ULL(i));
__GUEST_ASSERT(vector == GP_VECTOR,
"Expected #GP on XSETBV(0x%llx), supported XCR0 = %lx, got vector '0x%x'",
BIT_ULL(i), supported_xcr0, vector);
"Expected #GP on XSETBV(0x%llx), supported XCR0 = %lx, got %s",
BIT_ULL(i), supported_xcr0, ex_str(vector));
}
GUEST_DONE();