mirror of
https://github.com/DragonOS-Community/DragonOS.git
synced 2026-09-08 23:57:59 +08:00
perf(x86_64): accelerate logical CPU lookup (#2184)
* perf(x86_64): accelerate logical CPU lookup Replace the steady-state APIC MSR and CPUID based CPU lookup with a validated task-register selector decode. Extend the shared GDT so every supported CPU has a complete TSS descriptor, retain the bootstrap discovery fallback, and verify descriptor ownership before trusting the decoded logical ID. Keep VMX host state correct with per-CPU TR selectors by refreshing HOST_TR_SELECTOR and HOST_TR_BASE together. Add synchronized remote loaded-VMCS clearing, pin direct VMCS accesses against task migration, and snapshot exit information before returning to preemptible slow paths. Split MMU root preparation from EPTP submission so allocation remains preemptible while VMCS writes occur only after the current CPU and active VMCS are confirmed. This removes the dominant page-fault CPU-identification overhead without changing allocator zeroing semantics or adding workload-specific behavior. Validation: make kernel; RISC-V64 and LoongArch64 kernel checks; 2-vCPU QEMU boot; sched_affinity_test; page_fault_accounting_test; mlock_semantics_test; cargo fmt --check; git diff --check. Signed-off-by: longjin <longjin@dragonos.org> * fix(vmx): serialize active VMCS updates against IPIs Keep local interrupts disabled while updating the loaded-VMCS list and transitioning the per-CPU active VMCS. This prevents the remote-clear IPI from mutating current_vmcs through an exclusive reference while the ordinary load path still holds a shared reference. Complete any remote clear before entering the IRQ-disabled section, and publish the new owner only after the VMCS host state is initialized. The critical section therefore covers only per-CPU list and active-VMCS state without extending the synchronous remote-clear wait. Validation: cargo fmt --check; git diff --check; make kernel; three-role adversarial review covering concurrency, correctness, Linux semantics, lifecycle, and performance. Signed-off-by: longjin <longjin@dragonos.org> --------- Signed-off-by: longjin <longjin@dragonos.org>
This commit is contained in:
@@ -638,7 +638,9 @@ GDT_Table:
|
||||
.quad 0x00affb000000ffff // 6 用户64位代码段描述符 0x30
|
||||
.quad 0x00cf9a000000ffff // 7 内核32位代码段描述符 0x38
|
||||
.quad 0x00cf92000000ffff // 8 内核32位数据段描述符 0x40
|
||||
.fill 100, 8, 0 // 10-11 TSS(跳过了第9段) 重复十次填充8字节的空间,赋值为0 长模式下,每个TSS长度为128bit
|
||||
// Entry 9 is reserved. Entries 10..265 hold 128 per-CPU TSS descriptors.
|
||||
.fill 257, 8, 0
|
||||
.global GDT_END
|
||||
GDT_END:
|
||||
|
||||
.global GDT_POINTER
|
||||
@@ -733,4 +735,3 @@ gdt64_pointer:
|
||||
gdt64_pointer64:
|
||||
.short gdt64_pointer-gdt64-1
|
||||
.quad gdt64
|
||||
|
||||
@@ -3,7 +3,10 @@ use core::hint::spin_loop;
|
||||
use raw_cpuid::CpuId;
|
||||
use x86::msr::{rdmsr, IA32_APIC_BASE, IA32_X2APIC_APICID};
|
||||
|
||||
use crate::{arch::smp::SMP_BOOT_DATA, smp::cpu::ProcessorId};
|
||||
use crate::{
|
||||
arch::{process::table::TSSManager, smp::SMP_BOOT_DATA},
|
||||
smp::cpu::ProcessorId,
|
||||
};
|
||||
|
||||
/// 获取当前CPU的逻辑编号
|
||||
#[inline]
|
||||
@@ -12,6 +15,10 @@ pub fn current_cpu_id() -> ProcessorId {
|
||||
return ProcessorId::new(0);
|
||||
}
|
||||
|
||||
if let Some(cpu) = TSSManager::current_cpu_from_tr() {
|
||||
return cpu;
|
||||
}
|
||||
|
||||
let x2apic_id = current_x2apic_physical_id();
|
||||
if let Some(apic_id) = x2apic_id {
|
||||
if let Some(cpu_id) = phys_apic_id_to_cpu_id(apic_id) {
|
||||
|
||||
@@ -15,7 +15,7 @@ use crate::{
|
||||
entry::arch_setup_interrupt_gate,
|
||||
ipi::{
|
||||
arch_ipi_handler_init, send_ipi, IPI_NUM_FLUSH_TLB, IPI_NUM_KICK_CPU,
|
||||
IPI_NUM_STOP_CPU,
|
||||
IPI_NUM_LOADED_VMCS_CLEAR, IPI_NUM_STOP_CPU,
|
||||
},
|
||||
msi::{X86MsiAddrHi, X86MsiAddrLoNormal, X86MsiDataNormal, X86_MSI_BASE_ADDRESS_LOW},
|
||||
},
|
||||
@@ -268,6 +268,7 @@ pub fn arch_early_irq_init() -> Result<(), SystemError> {
|
||||
IPI_NUM_KICK_CPU,
|
||||
IPI_NUM_FLUSH_TLB,
|
||||
IPI_NUM_STOP_CPU,
|
||||
IPI_NUM_LOADED_VMCS_CLEAR,
|
||||
]);
|
||||
}
|
||||
return Ok(());
|
||||
|
||||
@@ -23,6 +23,7 @@ use super::TrapFrame;
|
||||
pub const IPI_NUM_KICK_CPU: IrqNumber = IrqNumber::new(200);
|
||||
pub const IPI_NUM_FLUSH_TLB: IrqNumber = IrqNumber::new(201);
|
||||
pub const IPI_NUM_STOP_CPU: IrqNumber = IrqNumber::new(202);
|
||||
pub const IPI_NUM_LOADED_VMCS_CLEAR: IrqNumber = IrqNumber::new(203);
|
||||
/// IPI的种类(架构相关,指定了向量号)
|
||||
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
|
||||
#[repr(u32)]
|
||||
@@ -273,6 +274,7 @@ pub fn arch_ipi_handler_init() {
|
||||
do_init_irq_handler(IPI_NUM_KICK_CPU);
|
||||
do_init_irq_handler(IPI_NUM_FLUSH_TLB);
|
||||
do_init_irq_handler(IPI_NUM_STOP_CPU);
|
||||
do_init_irq_handler(IPI_NUM_LOADED_VMCS_CLEAR);
|
||||
}
|
||||
|
||||
fn do_init_irq_handler(irq: IrqNumber) {
|
||||
@@ -304,6 +306,10 @@ impl IrqFlowHandler for X86_64IpiIrqFlowHandler {
|
||||
CurrentApic.send_eoi();
|
||||
stop_this_cpu();
|
||||
}
|
||||
IPI_NUM_LOADED_VMCS_CLEAR => {
|
||||
crate::arch::vm::vmx::VmxKvmFunc::handle_remote_loaded_vmcs_clear();
|
||||
CurrentApic.send_eoi();
|
||||
}
|
||||
_ => {
|
||||
error!("Unknown IPI: {}", irq.data());
|
||||
CurrentApic.send_eoi();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use core::cmp;
|
||||
use core::{cmp, ptr::addr_of};
|
||||
|
||||
use x86::{current::task::TaskStateSegment, segmentation::SegmentSelector, Ring};
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::{
|
||||
CurrentIrqArch,
|
||||
},
|
||||
exception::InterruptArch,
|
||||
mm::{percpu::PerCpu, VirtAddr},
|
||||
mm::{percpu::PerCpu, MemoryManagementArch, VirtAddr},
|
||||
process::ProcessManager,
|
||||
smp::core::smp_get_processor_id,
|
||||
};
|
||||
@@ -29,17 +29,23 @@ const IO_BITMAP_OFFSET_VALID: u16 = HARDWARE_TSS_SIZE as u16;
|
||||
const IO_BITMAP_TOTAL_BYTES: usize = IO_BITMAP_BYTES + IO_BITMAP_TERMINATOR_BYTES;
|
||||
const TSS_LIMIT: u16 = (HARDWARE_TSS_SIZE + IO_BITMAP_TOTAL_BYTES - 1) as u16;
|
||||
const IO_BITMAP_OFFSET_INVALID: u16 = TSS_LIMIT + 1;
|
||||
const TSS_GDT_BASE_INDEX: usize = 10;
|
||||
const TSS_GDT_ENTRY_STRIDE: usize = 2;
|
||||
const GDT_ENTRY_COUNT: usize =
|
||||
TSS_GDT_BASE_INDEX + PerCpu::MAX_CPU_NUM as usize * TSS_GDT_ENTRY_STRIDE;
|
||||
|
||||
const _: () = assert!(HARDWARE_TSS_SIZE <= u16::MAX as usize);
|
||||
const _: () = assert!(HARDWARE_TSS_SIZE + IO_BITMAP_TOTAL_BYTES <= u16::MAX as usize);
|
||||
const _: () = assert!(TSS_LIMIT as usize <= 0x000f_ffff);
|
||||
const _: () = assert!(GDT_ENTRY_COUNT <= u16::MAX as usize);
|
||||
|
||||
static mut TSS_MANAGER: TSSManager = TSSManager {
|
||||
tss: [DragonOsTss::new(); PerCpu::MAX_CPU_NUM as usize],
|
||||
};
|
||||
|
||||
extern "C" {
|
||||
static mut GDT_Table: [u64; 512];
|
||||
static mut GDT_Table: [u64; GDT_ENTRY_COUNT];
|
||||
static GDT_END: u8;
|
||||
}
|
||||
|
||||
/// 切换fs和gs段寄存器
|
||||
@@ -106,6 +112,84 @@ pub struct TSSManager {
|
||||
}
|
||||
|
||||
impl TSSManager {
|
||||
#[inline]
|
||||
fn selector_index(cpu: u32) -> Option<usize> {
|
||||
let cpu = usize::try_from(cpu).ok()?;
|
||||
if cpu >= PerCpu::MAX_CPU_NUM as usize {
|
||||
return None;
|
||||
}
|
||||
|
||||
TSS_GDT_BASE_INDEX.checked_add(cpu.checked_mul(TSS_GDT_ENTRY_STRIDE)?)
|
||||
}
|
||||
|
||||
/// Returns the logical CPU encoded by a DragonOS-owned task register.
|
||||
///
|
||||
/// The descriptor identity check rejects selectors left by firmware or a
|
||||
/// bootloader. Callers must retain their bootstrap fallback until SMP boot
|
||||
/// data is initialized and this CPU has loaded its DragonOS TSS.
|
||||
#[inline]
|
||||
pub fn current_cpu_from_tr() -> Option<crate::smp::cpu::ProcessorId> {
|
||||
let selector = unsafe { x86::task::tr() }.bits();
|
||||
if selector & 0x7 != 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let index = usize::from(selector >> 3);
|
||||
let relative = index.checked_sub(TSS_GDT_BASE_INDEX)?;
|
||||
if relative % TSS_GDT_ENTRY_STRIDE != 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let cpu = relative / TSS_GDT_ENTRY_STRIDE;
|
||||
if cpu >= PerCpu::MAX_CPU_NUM as usize {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (low, high) = unsafe {
|
||||
let gdt = Self::gdt_virtual_base().data() as *const u64;
|
||||
(
|
||||
gdt.add(index).read_volatile(),
|
||||
gdt.add(index + 1).read_volatile(),
|
||||
)
|
||||
};
|
||||
let descriptor_type = (low >> 40) & 0xf;
|
||||
let system = (low >> 44) & 1;
|
||||
let present = (low >> 47) & 1;
|
||||
if descriptor_type != 0xb || system != 0 || present == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let base = ((low >> 16) & 0xffff)
|
||||
| (((low >> 32) & 0xff) << 16)
|
||||
| (((low >> 56) & 0xff) << 24)
|
||||
| ((high & 0xffff_ffff) << 32);
|
||||
let expected = unsafe { addr_of!(TSS_MANAGER.tss).cast::<DragonOsTss>().add(cpu) as u64 };
|
||||
if base != expected {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(crate::smp::cpu::ProcessorId::new(cpu as u32))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn gdt_entry_count() -> usize {
|
||||
let start = addr_of!(GDT_Table) as usize;
|
||||
let end = addr_of!(GDT_END) as usize;
|
||||
let bytes = end.checked_sub(start).expect("GDT_END precedes GDT_Table");
|
||||
assert_eq!(bytes % core::mem::size_of::<u64>(), 0);
|
||||
let entries = bytes / core::mem::size_of::<u64>();
|
||||
assert_eq!(
|
||||
entries, GDT_ENTRY_COUNT,
|
||||
"assembly and Rust GDT sizes differ"
|
||||
);
|
||||
entries
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn gdt_virtual_base() -> VirtAddr {
|
||||
VirtAddr::new(addr_of!(GDT_Table) as usize + crate::arch::MMArch::PHYS_OFFSET)
|
||||
}
|
||||
|
||||
/// 获取当前CPU的TSS
|
||||
pub unsafe fn current_tss() -> &'static mut DragonOsTss {
|
||||
&mut TSS_MANAGER.tss[smp_get_processor_id().data() as usize]
|
||||
@@ -113,7 +197,8 @@ impl TSSManager {
|
||||
|
||||
/// 加载当前CPU的TSS
|
||||
pub unsafe fn load_tr() {
|
||||
let index = (10 + smp_get_processor_id().data() * 2) as u16;
|
||||
let cpu = smp_get_processor_id();
|
||||
let index = Self::selector_index(cpu.data()).expect("CPU ID exceeds the TSS table") as u16;
|
||||
let selector = SegmentSelector::new(index, Ring::Ring0);
|
||||
|
||||
Self::set_tss_descriptor(index, VirtAddr::new(Self::current_tss() as *mut _ as usize));
|
||||
@@ -141,12 +226,12 @@ impl TSSManager {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(static_mut_refs)]
|
||||
unsafe fn set_tss_descriptor(index: u16, vaddr: VirtAddr) {
|
||||
let limit = TSS_LIMIT as u64;
|
||||
let gdt_vaddr = VirtAddr::new(&GDT_Table as *const _ as usize);
|
||||
|
||||
let gdt: &mut [u64] = core::slice::from_raw_parts_mut(gdt_vaddr.data() as *mut u64, 512);
|
||||
let entries = Self::gdt_entry_count();
|
||||
assert!((index as usize + 1) < entries);
|
||||
let gdt =
|
||||
core::slice::from_raw_parts_mut(Self::gdt_virtual_base().data() as *mut u64, entries);
|
||||
|
||||
let vaddr = vaddr.data() as u64;
|
||||
gdt[index as usize] = (limit & 0xffff)
|
||||
|
||||
@@ -218,6 +218,9 @@ pub trait KvmFunc: Send + Sync + Debug {
|
||||
|
||||
fn handle_exit_irqoff(&self, vcpu: &mut VirtCpu);
|
||||
|
||||
/// Caches VM-exit fields that are consumed after migration is allowed again.
|
||||
fn cache_exit_info(&self, vcpu: &mut VirtCpu);
|
||||
|
||||
fn handle_exit(
|
||||
&self,
|
||||
vcpu: &mut VirtCpu,
|
||||
|
||||
@@ -35,6 +35,7 @@ use crate::{
|
||||
},
|
||||
},
|
||||
mm::VirtAddr,
|
||||
process::preempt::PreemptGuard,
|
||||
smp::{core::smp_get_processor_id, cpu::ProcessorId},
|
||||
virt::vm::{
|
||||
kvm_host::{
|
||||
@@ -634,8 +635,9 @@ impl VirtCpu {
|
||||
|
||||
self.arch.pat = MSR_IA32_CR_PAT_DEFAULT;
|
||||
|
||||
self.load();
|
||||
let preempt_guard = self.load();
|
||||
self.vcpu_reset(vm, false)?;
|
||||
drop(preempt_guard);
|
||||
self.arch.kvm_init_mmu();
|
||||
|
||||
Ok(())
|
||||
@@ -652,7 +654,7 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
pub fn run(&mut self) -> Result<usize, SystemError> {
|
||||
self.load();
|
||||
let preempt_guard = self.load();
|
||||
|
||||
if unlikely(self.arch.mp_state == MutilProcessorState::Uninitialized) {
|
||||
todo!()
|
||||
@@ -672,6 +674,8 @@ impl VirtCpu {
|
||||
self.kvm_set_cr8(self.kvm_run().cr8);
|
||||
}
|
||||
|
||||
drop(preempt_guard);
|
||||
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#11174 - 11196
|
||||
|
||||
if self.kvm_run().immediate_exit != 0 {
|
||||
@@ -699,196 +703,219 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
fn enter_guest(&mut self, vm: &Vm) -> Result<(), SystemError> {
|
||||
let req_immediate_exit = false;
|
||||
|
||||
warn!("request {:?}", self.request);
|
||||
if !self.request.is_empty() {
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_VM_DEAD) {
|
||||
return Err(SystemError::EIO);
|
||||
}
|
||||
|
||||
// TODO: kvm_dirty_ring_check_request
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MMU_FREE_OBSOLETE_ROOTS) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MIGRATE_TIMER) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MASTERCLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_GLOBAL_CLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_CLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MMU_SYNC) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_LOAD_MMU_PGD) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_TLB_FLUSH) {
|
||||
self.flush_tlb_all();
|
||||
}
|
||||
|
||||
self.service_local_tlb_flush_requests();
|
||||
|
||||
// TODO: KVM_REQ_HV_TLB_FLUSH) && kvm_hv_vcpu_flush_tlb(vcpu)
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_REPORT_TPR_ACCESS) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_TRIPLE_FAULT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_STEAL_UPDATE) {
|
||||
// todo!()
|
||||
warn!("VirtCpuRequest::KVM_REQ_STEAL_UPDATE TODO!");
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_SMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_NMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_PMU) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_PMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_IOAPIC_EOI_EXIT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_SCAN_IOAPIC) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_LOAD_EOI_EXITMAP) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APIC_PAGE_RELOAD) {
|
||||
// todo!()
|
||||
warn!("VirtCpuRequest::KVM_REQ_APIC_PAGE_RELOAD TODO!");
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_CRASH) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_RESET) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_EXIT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_STIMER) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APICV_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APF_READY) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MSR_FILTER_CHANGED) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_UPDATE_CPU_DIRTY_LOGGING) {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10661
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_EVENT) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
self.kvm_mmu_reload(vm)?;
|
||||
|
||||
x86_kvm_ops().prepare_switch_to_guest(self);
|
||||
// warn!(
|
||||
// "mode {:?} req {:?} mode_cond {} !is_empty {} cond {}",
|
||||
// self.mode,
|
||||
// self.request,
|
||||
// self.mode == VcpuMode::ExitingGuestMode,
|
||||
// !self.request.is_empty(),
|
||||
// (self.mode == VcpuMode::ExitingGuestMode) || (!self.request.is_empty())
|
||||
// );
|
||||
warn!(
|
||||
"req bit {} empty bit {}",
|
||||
self.request.bits,
|
||||
VirtCpuRequest::empty().bits
|
||||
);
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10730
|
||||
if self.mode == VcpuMode::ExitingGuestMode || !self.request.is_empty() {
|
||||
self.mode = VcpuMode::OutsideGuestMode;
|
||||
return Err(SystemError::EINVAL);
|
||||
}
|
||||
|
||||
if req_immediate_exit {
|
||||
self.request(VirtCpuRequest::KVM_REQ_EVENT);
|
||||
todo!();
|
||||
}
|
||||
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10749 - 10766
|
||||
|
||||
let exit_fastpath;
|
||||
let mut mmu_pgd_needs_commit = false;
|
||||
loop {
|
||||
exit_fastpath = x86_kvm_ops().vcpu_run(self);
|
||||
if likely(exit_fastpath != ExitFastpathCompletion::ExitHandled) {
|
||||
break;
|
||||
let req_immediate_exit = false;
|
||||
|
||||
warn!("request {:?}", self.request);
|
||||
if !self.request.is_empty() {
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_VM_DEAD) {
|
||||
return Err(SystemError::EIO);
|
||||
}
|
||||
|
||||
// TODO: kvm_dirty_ring_check_request
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MMU_FREE_OBSOLETE_ROOTS) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MIGRATE_TIMER) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MASTERCLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_GLOBAL_CLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_CLOCK_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MMU_SYNC) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_LOAD_MMU_PGD) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_TLB_FLUSH) {
|
||||
self.flush_tlb_all();
|
||||
}
|
||||
|
||||
self.service_local_tlb_flush_requests();
|
||||
|
||||
// TODO: KVM_REQ_HV_TLB_FLUSH) && kvm_hv_vcpu_flush_tlb(vcpu)
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_REPORT_TPR_ACCESS) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_TRIPLE_FAULT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_STEAL_UPDATE) {
|
||||
// todo!()
|
||||
warn!("VirtCpuRequest::KVM_REQ_STEAL_UPDATE TODO!");
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_SMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_NMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_PMU) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_PMI) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_IOAPIC_EOI_EXIT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_SCAN_IOAPIC) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_LOAD_EOI_EXITMAP) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APIC_PAGE_RELOAD) {
|
||||
// todo!()
|
||||
warn!("VirtCpuRequest::KVM_REQ_APIC_PAGE_RELOAD TODO!");
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_CRASH) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_RESET) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_EXIT) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_HV_STIMER) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APICV_UPDATE) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_APF_READY) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_MSR_FILTER_CHANGED) {
|
||||
todo!()
|
||||
}
|
||||
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_UPDATE_CPU_DIRTY_LOGGING) {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
||||
todo!();
|
||||
}
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10661
|
||||
if self.check_request(VirtCpuRequest::KVM_REQ_EVENT) {
|
||||
// TODO
|
||||
}
|
||||
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10799 - 10814
|
||||
mmu_pgd_needs_commit |= self.kvm_mmu_reload(vm)?;
|
||||
|
||||
self.arch.last_vmentry_cpu = self.cpu;
|
||||
let preempt_guard = PreemptGuard::new();
|
||||
let current_cpu = smp_get_processor_id();
|
||||
if current_cpu != self.cpu {
|
||||
self.arch_vcpu_load(current_cpu);
|
||||
drop(preempt_guard);
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: last_guest_tsc
|
||||
// CPU identity alone is insufficient: another vCPU may have replaced
|
||||
// this CPU's active VMCS while this task was preempted.
|
||||
x86_kvm_ops().vcpu_load(self, current_cpu);
|
||||
|
||||
self.mode = VcpuMode::OutsideGuestMode;
|
||||
if mmu_pgd_needs_commit {
|
||||
self.kvm_mmu_load_pgd(vm);
|
||||
}
|
||||
|
||||
barrier::mfence();
|
||||
x86_kvm_ops().prepare_switch_to_guest(self);
|
||||
// warn!(
|
||||
// "mode {:?} req {:?} mode_cond {} !is_empty {} cond {}",
|
||||
// self.mode,
|
||||
// self.request,
|
||||
// self.mode == VcpuMode::ExitingGuestMode,
|
||||
// !self.request.is_empty(),
|
||||
// (self.mode == VcpuMode::ExitingGuestMode) || (!self.request.is_empty())
|
||||
// );
|
||||
warn!(
|
||||
"req bit {} empty bit {}",
|
||||
self.request.bits,
|
||||
VirtCpuRequest::empty().bits
|
||||
);
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10730
|
||||
if self.mode == VcpuMode::ExitingGuestMode || !self.request.is_empty() {
|
||||
self.mode = VcpuMode::OutsideGuestMode;
|
||||
drop(preempt_guard);
|
||||
return Err(SystemError::EINVAL);
|
||||
}
|
||||
|
||||
// TODO: xfd
|
||||
if req_immediate_exit {
|
||||
self.request(VirtCpuRequest::KVM_REQ_EVENT);
|
||||
todo!();
|
||||
}
|
||||
|
||||
x86_kvm_ops().handle_exit_irqoff(self);
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10749 - 10766
|
||||
|
||||
// todo: xfd
|
||||
let exit_fastpath;
|
||||
loop {
|
||||
exit_fastpath = x86_kvm_ops().vcpu_run(self);
|
||||
if likely(exit_fastpath != ExitFastpathCompletion::ExitHandled) {
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO: 一些中断或者tsc操作
|
||||
todo!();
|
||||
}
|
||||
|
||||
match x86_kvm_ops().handle_exit(self, vm, exit_fastpath) {
|
||||
Err(err) => return Err(err),
|
||||
Ok(_) => Ok(()),
|
||||
// TODO: https://code.dragonos.org.cn/xref/linux-6.6.21/arch/x86/kvm/x86.c#10799 - 10814
|
||||
|
||||
self.arch.last_vmentry_cpu = self.cpu;
|
||||
|
||||
// TODO: last_guest_tsc
|
||||
|
||||
self.mode = VcpuMode::OutsideGuestMode;
|
||||
|
||||
barrier::mfence();
|
||||
|
||||
// TODO: xfd
|
||||
|
||||
x86_kvm_ops().handle_exit_irqoff(self);
|
||||
x86_kvm_ops().cache_exit_info(self);
|
||||
|
||||
drop(preempt_guard);
|
||||
|
||||
// todo: xfd
|
||||
|
||||
// TODO: Handle the remaining interrupt and TSC state updates.
|
||||
|
||||
return match x86_kvm_ops().handle_exit(self, vm, exit_fastpath) {
|
||||
Err(err) => return Err(err),
|
||||
Ok(_) => Ok(()),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -946,8 +973,10 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn load(&mut self) {
|
||||
self.arch_vcpu_load(smp_get_processor_id())
|
||||
fn load(&mut self) -> PreemptGuard {
|
||||
let preempt_guard = PreemptGuard::new();
|
||||
self.arch_vcpu_load(smp_get_processor_id());
|
||||
preempt_guard
|
||||
}
|
||||
|
||||
fn arch_vcpu_load(&mut self, cpu: ProcessorId) {
|
||||
@@ -1151,7 +1180,7 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
pub fn get_regs(&mut self) -> KvmCommonRegs {
|
||||
self.load();
|
||||
let _preempt_guard = self.load();
|
||||
return self._get_regs();
|
||||
}
|
||||
|
||||
@@ -1179,7 +1208,7 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
pub fn get_segment_regs(&mut self) -> UapiKvmSegmentRegs {
|
||||
self.load();
|
||||
let _preempt_guard = self.load();
|
||||
return self._get_segment_regs();
|
||||
}
|
||||
|
||||
@@ -1251,7 +1280,7 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
pub fn set_segment_regs(&mut self, sregs: &mut UapiKvmSegmentRegs) -> Result<(), SystemError> {
|
||||
self.load();
|
||||
let _preempt_guard = self.load();
|
||||
self._set_segmenet_regs(&self.kvm().lock(), sregs)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1426,7 +1455,7 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
pub fn set_regs(&mut self, regs: &KvmCommonRegs) -> Result<(), SystemError> {
|
||||
self.load();
|
||||
let _preempt_guard = self.load();
|
||||
self._set_regs(regs);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -535,12 +535,14 @@ impl VirtCpuArch {
|
||||
}
|
||||
|
||||
impl VirtCpu {
|
||||
pub fn kvm_mmu_reload(&mut self, vm: &Vm) -> Result<(), SystemError> {
|
||||
/// Prepares an MMU root without touching the CPU-local active VMCS.
|
||||
pub fn kvm_mmu_reload(&mut self, vm: &Vm) -> Result<bool, SystemError> {
|
||||
if likely(self.arch.mmu().root.hpa != KvmMmu::INVALID_PAGE) {
|
||||
return Ok(());
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
return self.kvm_mmu_load(vm);
|
||||
self.kvm_mmu_load(vm)?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub fn kvm_mmu_load(&mut self, vm: &Vm) -> Result<(), SystemError> {
|
||||
@@ -556,8 +558,6 @@ impl VirtCpu {
|
||||
|
||||
// TODO: kvm_mmu_sync_roots
|
||||
|
||||
self.kvm_mmu_load_pgd(vm);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,12 @@ use crate::{arch::vm::mmu::kvm_mmu::PAGE_SHIFT, mm::page::EntryFlags};
|
||||
use alloc::sync::Arc;
|
||||
use core::{intrinsics::unlikely, ops::Index};
|
||||
use log::{debug, warn};
|
||||
use x86::vmx::vmcs::{guest, host};
|
||||
|
||||
use system_error::SystemError;
|
||||
|
||||
use crate::{
|
||||
arch::{
|
||||
vm::{
|
||||
asm::VmxAsm,
|
||||
kvm_host::{EmulType, KVM_PFN_NOSLOT},
|
||||
mmu::kvm_mmu::{PFRet, PageLevel},
|
||||
mtrr::kvm_mtrr_check_gfn_range_consistency,
|
||||
@@ -361,9 +359,6 @@ impl VirtCpu {
|
||||
}
|
||||
|
||||
// 尝试将 GFN 转换为 PFN
|
||||
let guest_cr3 = VmxAsm::vmx_vmread(guest::CR3);
|
||||
let host_cr3 = VmxAsm::vmx_vmread(host::CR3);
|
||||
debug!("guest_cr3={:x}, host_cr3={:x}", guest_cr3, host_cr3);
|
||||
page_fault.pfn = __gfn_to_pfn_memslot(
|
||||
Some(&slot),
|
||||
page_fault.gfn,
|
||||
|
||||
@@ -1,13 +1,8 @@
|
||||
use crate::virt::vm::kvm_host::{vcpu::VirtCpu, Vm};
|
||||
use bitfield_struct::bitfield;
|
||||
use system_error::SystemError;
|
||||
use x86::vmx::vmcs::{guest, ro};
|
||||
|
||||
use crate::{
|
||||
arch::vm::asm::{IntrInfo, VmxAsm},
|
||||
virt::vm::kvm_host::{vcpu::VirtCpu, Vm},
|
||||
};
|
||||
|
||||
use super::{ept::EptViolationExitQual, vmx_info, PageFaultErr};
|
||||
use super::{ept::EptViolationExitQual, PageFaultErr};
|
||||
extern crate num_traits;
|
||||
|
||||
#[bitfield(u32)]
|
||||
@@ -289,14 +284,7 @@ impl VmxExitHandlers {
|
||||
// 在下一次 VM 进入之前必须设置 "blocked by NMI" 位。
|
||||
// 有一些错误可能会导致该位未被设置:
|
||||
// AAK134, BY25。
|
||||
let vmx = vcpu.vmx();
|
||||
if vmx.idt_vectoring_info.bits() & IntrInfo::INTR_INFO_VALID_MASK.bits() != 0
|
||||
&& vmx_info().enable_vnmi
|
||||
&& exit_qualification & IntrInfo::INTR_INFO_UNBLOCK_NMI.bits() as u64 != 0
|
||||
{
|
||||
VmxAsm::vmx_vmwrite(guest::INTERRUPTIBILITY_STATE, 0x8); //GUEST_INTR_STATE_NMI
|
||||
}
|
||||
let gpa = VmxAsm::vmx_vmread(ro::GUEST_PHYSICAL_ADDR_FULL);
|
||||
let gpa = vcpu.vmx().get_exit_gpa();
|
||||
//let exit_qualification = VmxAsm::vmx_vmread(ro::EXIT_QUALIFICATION);
|
||||
// trace_kvm_page_fault(vcpu, gpa, exit_qualification);//
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ use log::warn;
|
||||
use x86_64::registers::control::Cr3Flags;
|
||||
use x86_64::structures::paging::PhysFrame;
|
||||
|
||||
use crate::arch::interrupt::ipi::{send_ipi, IPI_NUM_LOADED_VMCS_CLEAR};
|
||||
use crate::arch::process::table::USER_DS;
|
||||
use crate::arch::vm::mmu::kvm_mmu::KvmMmu;
|
||||
use crate::arch::vm::uapi::kvm_exit;
|
||||
@@ -17,6 +18,10 @@ use crate::arch::vm::uapi::{
|
||||
AC_VECTOR, BP_VECTOR, DB_VECTOR, GP_VECTOR, MC_VECTOR, NM_VECTOR, PF_VECTOR, UD_VECTOR,
|
||||
};
|
||||
use crate::arch::vm::vmx::vmcs::VmcsIntrHelper;
|
||||
use crate::exception::{
|
||||
ipi::{IpiKind, IpiTarget},
|
||||
HardwareIrqNumber,
|
||||
};
|
||||
use crate::libs::spinlock::SpinLockGuard;
|
||||
use crate::mm::VirtAddr;
|
||||
use crate::process::ProcessManager;
|
||||
@@ -36,7 +41,10 @@ use crate::{
|
||||
percpu::{PerCpu, PerCpuVar},
|
||||
MemoryManagementArch,
|
||||
},
|
||||
smp::{core::smp_get_processor_id, cpu::ProcessorId},
|
||||
smp::{
|
||||
core::smp_get_processor_id,
|
||||
cpu::{smp_cpu_manager, ProcessorId},
|
||||
},
|
||||
virt::vm::{kvm_dev::kvm_init, kvm_host::vcpu::VirtCpu, user_api::UapiKvmSegment},
|
||||
};
|
||||
use alloc::{alloc::Global, boxed::Box, collections::LinkedList, sync::Arc, vec::Vec};
|
||||
@@ -311,6 +319,20 @@ impl KvmInitFunc for VmxKvmInitFunc {
|
||||
#[derive(Debug)]
|
||||
pub struct VmxKvmFunc;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteLoadedVmcsClear {
|
||||
loaded_vmcs: Arc<LockedLoadedVmcs>,
|
||||
vmcs: Arc<self::vmcs::LockedVMControlStructure>,
|
||||
shadow_vmcs: Option<Arc<self::vmcs::LockedVMControlStructure>>,
|
||||
launched: bool,
|
||||
owner: ProcessorId,
|
||||
}
|
||||
|
||||
static REMOTE_LOADED_VMCS_CLEAR_SERIAL: SpinLock<()> = SpinLock::new(());
|
||||
static REMOTE_LOADED_VMCS_CLEAR_REQUEST: SpinLock<Option<RemoteLoadedVmcsClear>> =
|
||||
SpinLock::new(None);
|
||||
static REMOTE_LOADED_VMCS_CLEAR_DONE: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub struct VmxKvmFuncConfig {
|
||||
pub have_set_apic_access_page_addr: bool,
|
||||
pub have_update_cr8_intercept: bool,
|
||||
@@ -329,29 +351,31 @@ impl VmxKvmFunc {
|
||||
_buddy: Option<Arc<LockedLoadedVmcs>>,
|
||||
) {
|
||||
let vmx = vcpu.vmx();
|
||||
let already_loaded = vmx.loaded_vmcs.lock().cpu == cpu;
|
||||
let already_loaded = vmx.loaded_vmcs.owner_cpu() == cpu;
|
||||
|
||||
if !already_loaded {
|
||||
Self::loaded_vmcs_clear(&vmx.loaded_vmcs);
|
||||
let _irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() };
|
||||
|
||||
current_loaded_vmcs_list_mut().push_back(vmx.loaded_vmcs.clone());
|
||||
}
|
||||
|
||||
if let Some(prev) = current_vmcs() {
|
||||
{
|
||||
// The remote-clear IPI accesses both per-CPU structures. Keep the
|
||||
// list update and active-VMCS transition in one IRQ-safe section.
|
||||
let _irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() };
|
||||
|
||||
if !already_loaded {
|
||||
current_loaded_vmcs_list_mut().push_back(vmx.loaded_vmcs.clone());
|
||||
}
|
||||
|
||||
let vmcs = vmx.loaded_vmcs.lock().vmcs.clone();
|
||||
if !Arc::ptr_eq(&vmcs, prev) {
|
||||
if !current_vmcs()
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(current, &vmcs))
|
||||
{
|
||||
VmxAsm::vmcs_load(vmcs.phys_addr());
|
||||
*current_vmcs_mut() = Some(vmcs);
|
||||
|
||||
// TODO:buddy barrier?
|
||||
}
|
||||
} else {
|
||||
let vmcs = vmx.loaded_vmcs.lock().vmcs.clone();
|
||||
VmxAsm::vmcs_load(vmcs.phys_addr());
|
||||
*current_vmcs_mut() = Some(vmcs);
|
||||
|
||||
// TODO:buddy barrier?
|
||||
}
|
||||
|
||||
if !already_loaded {
|
||||
@@ -361,7 +385,6 @@ impl VmxKvmFunc {
|
||||
x86::dtables::sgdt(&mut pseudo_descriptpr);
|
||||
};
|
||||
|
||||
vmx.loaded_vmcs.lock().cpu = cpu;
|
||||
let id = vmx.loaded_vmcs.lock().vmcs.lock().revision_id();
|
||||
debug!(
|
||||
"revision_id {id} req {:?}",
|
||||
@@ -369,12 +392,14 @@ impl VmxKvmFunc {
|
||||
);
|
||||
vcpu.request(VirtCpuRequest::KVM_REQ_TLB_FLUSH);
|
||||
|
||||
let tr_selector = unsafe { x86::task::tr().bits() };
|
||||
VmxAsm::vmx_vmwrite(host::TR_SELECTOR, tr_selector as u64);
|
||||
VmxAsm::vmx_vmwrite(
|
||||
host::TR_BASE,
|
||||
KvmX86Asm::get_segment_base(
|
||||
pseudo_descriptpr.base,
|
||||
pseudo_descriptpr.limit,
|
||||
unsafe { x86::task::tr().bits() },
|
||||
tr_selector,
|
||||
),
|
||||
);
|
||||
|
||||
@@ -383,16 +408,24 @@ impl VmxKvmFunc {
|
||||
VmxAsm::vmx_vmwrite(host::IA32_SYSENTER_ESP, unsafe {
|
||||
rdmsr(msr::IA32_SYSENTER_ESP)
|
||||
});
|
||||
|
||||
// Publish ownership only after the VMCS and host state are ready.
|
||||
vcpu.vmx().loaded_vmcs.set_owner_cpu(cpu);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn loaded_vmcs_clear(loaded_vmcs: &Arc<LockedLoadedVmcs>) {
|
||||
let mut guard = loaded_vmcs.lock();
|
||||
if guard.cpu == ProcessorId::INVALID {
|
||||
let owner = loaded_vmcs.owner_cpu();
|
||||
if owner == ProcessorId::INVALID {
|
||||
return;
|
||||
}
|
||||
|
||||
if guard.cpu == smp_get_processor_id() {
|
||||
if owner == smp_get_processor_id() {
|
||||
let _irq_guard = unsafe { CurrentIrqArch::save_and_disable_irq() };
|
||||
let mut guard = loaded_vmcs.lock();
|
||||
if loaded_vmcs.owner_cpu() != owner {
|
||||
return;
|
||||
}
|
||||
if let Some(vmcs) = current_vmcs() {
|
||||
if Arc::ptr_eq(vmcs, &guard.vmcs) {
|
||||
*current_vmcs_mut() = None;
|
||||
@@ -407,16 +440,92 @@ impl VmxKvmFunc {
|
||||
}
|
||||
}
|
||||
|
||||
let _ = current_loaded_vmcs_list_mut().extract_if(|x| Arc::ptr_eq(x, loaded_vmcs));
|
||||
current_loaded_vmcs_list_mut()
|
||||
.extract_if(|x| Arc::ptr_eq(x, loaded_vmcs))
|
||||
.for_each(drop);
|
||||
|
||||
guard.cpu = ProcessorId::INVALID;
|
||||
guard.launched = false;
|
||||
loaded_vmcs.set_owner_cpu(ProcessorId::INVALID);
|
||||
} else {
|
||||
// 交由对应cpu处理
|
||||
todo!()
|
||||
Self::remote_loaded_vmcs_clear(loaded_vmcs, owner);
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_loaded_vmcs_clear(loaded_vmcs: &Arc<LockedLoadedVmcs>, owner: ProcessorId) {
|
||||
let _serial = REMOTE_LOADED_VMCS_CLEAR_SERIAL.lock();
|
||||
if loaded_vmcs.owner_cpu() != owner {
|
||||
return;
|
||||
}
|
||||
assert!(
|
||||
smp_cpu_manager().is_online_cpu(owner),
|
||||
"VMCS owner CPU is offline"
|
||||
);
|
||||
|
||||
let request = {
|
||||
let guard = loaded_vmcs.lock();
|
||||
if loaded_vmcs.owner_cpu() != owner {
|
||||
return;
|
||||
}
|
||||
RemoteLoadedVmcsClear {
|
||||
loaded_vmcs: loaded_vmcs.clone(),
|
||||
vmcs: guard.vmcs.clone(),
|
||||
shadow_vmcs: guard.shadow_vmcs.clone(),
|
||||
launched: guard.launched,
|
||||
owner,
|
||||
}
|
||||
};
|
||||
REMOTE_LOADED_VMCS_CLEAR_DONE.store(false, Ordering::Relaxed);
|
||||
{
|
||||
let mut slot = REMOTE_LOADED_VMCS_CLEAR_REQUEST.lock();
|
||||
assert!(slot.is_none());
|
||||
*slot = Some(request);
|
||||
}
|
||||
|
||||
send_ipi(
|
||||
IpiKind::SpecVector(HardwareIrqNumber::new(IPI_NUM_LOADED_VMCS_CLEAR.data())),
|
||||
IpiTarget::Specified(owner),
|
||||
);
|
||||
while !REMOTE_LOADED_VMCS_CLEAR_DONE.load(Ordering::Acquire) {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
*REMOTE_LOADED_VMCS_CLEAR_REQUEST.lock() = None;
|
||||
|
||||
assert_eq!(loaded_vmcs.owner_cpu(), ProcessorId::INVALID);
|
||||
loaded_vmcs.lock().launched = false;
|
||||
}
|
||||
|
||||
/// Clears a loaded VMCS on its owning CPU from the dedicated IPI context.
|
||||
pub fn handle_remote_loaded_vmcs_clear() {
|
||||
let request = REMOTE_LOADED_VMCS_CLEAR_REQUEST.lock().clone();
|
||||
let Some(request) = request else {
|
||||
return;
|
||||
};
|
||||
if request.owner != smp_get_processor_id()
|
||||
|| request.loaded_vmcs.owner_cpu() != request.owner
|
||||
{
|
||||
REMOTE_LOADED_VMCS_CLEAR_DONE.store(true, Ordering::Release);
|
||||
return;
|
||||
}
|
||||
|
||||
if current_vmcs()
|
||||
.as_ref()
|
||||
.is_some_and(|vmcs| Arc::ptr_eq(vmcs, &request.vmcs))
|
||||
{
|
||||
*current_vmcs_mut() = None;
|
||||
}
|
||||
VmxAsm::vmclear(request.vmcs.phys_addr());
|
||||
if request.launched {
|
||||
if let Some(shadow) = &request.shadow_vmcs {
|
||||
VmxAsm::vmclear(shadow.phys_addr());
|
||||
}
|
||||
}
|
||||
current_loaded_vmcs_list_mut()
|
||||
.extract_if(|entry| Arc::ptr_eq(entry, &request.loaded_vmcs))
|
||||
.for_each(drop);
|
||||
request.loaded_vmcs.set_owner_cpu(ProcessorId::INVALID);
|
||||
REMOTE_LOADED_VMCS_CLEAR_DONE.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub fn seg_setup(&self, seg: VcpuSegment) {
|
||||
let seg_field = &KVM_VMX_SEGMENT_FIELDS[seg as usize];
|
||||
|
||||
@@ -1210,6 +1319,31 @@ impl KvmFunc for VmxKvmFunc {
|
||||
}
|
||||
}
|
||||
|
||||
fn cache_exit_info(&self, vcpu: &mut VirtCpu) {
|
||||
if vcpu.vmx().emulation_required || vcpu.vmx().exit_reason.failed_vmentry() {
|
||||
return;
|
||||
}
|
||||
|
||||
if VmxExitReasonBasic::from(vcpu.vmx().exit_reason.basic())
|
||||
== VmxExitReasonBasic::EPT_VIOLATION
|
||||
{
|
||||
let qualification = VmxAsm::vmx_vmread(ro::EXIT_QUALIFICATION);
|
||||
let gpa = VmxAsm::vmx_vmread(ro::GUEST_PHYSICAL_ADDR_FULL);
|
||||
vcpu.vmx_mut().exit_qualification = qualification;
|
||||
vcpu.vmx_mut().exit_gpa = gpa;
|
||||
vcpu.arch
|
||||
.mark_register_available(KvmReg::VcpuExregExitInfo1);
|
||||
|
||||
if vcpu.vmx().idt_vectoring_info.bits() & IntrInfo::INTR_INFO_VALID_MASK.bits() == 0
|
||||
&& vmx_info().enable_vnmi
|
||||
&& qualification & IntrInfo::INTR_INFO_UNBLOCK_NMI.bits() as u64 != 0
|
||||
{
|
||||
let state = VmxAsm::vmx_vmread(guest::INTERRUPTIBILITY_STATE);
|
||||
VmxAsm::vmx_vmwrite(guest::INTERRUPTIBILITY_STATE, state | (1 << 3));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_exit(
|
||||
//vmx_handle_exit
|
||||
&self,
|
||||
@@ -1904,6 +2038,7 @@ impl Vmx {
|
||||
}
|
||||
|
||||
/// 打印VMCS信息用于debug
|
||||
#[allow(dead_code)]
|
||||
pub fn dump_vmcs(&self, vcpu: &VirtCpu) {
|
||||
let vmentry_ctl = unsafe {
|
||||
EntryControls::from_bits_unchecked(self.vmread(control::VMENTRY_CONTROLS) as u32)
|
||||
@@ -2218,6 +2353,7 @@ impl Vmx {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn dump_sel(&self, name: &'static str, sel: u32) {
|
||||
error!(
|
||||
"{name} sel = 0x{:x}, attr = 0x{:x}, limit = 0x{:x}, base = 0x{:x}",
|
||||
@@ -2228,6 +2364,7 @@ impl Vmx {
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn dump_dtsel(&self, name: &'static str, limit: u32) {
|
||||
error!(
|
||||
"{name} limit = 0x{:x}, base = 0x{:x}",
|
||||
@@ -2236,6 +2373,7 @@ impl Vmx {
|
||||
);
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn dump_msrs(&self, name: &'static str, msr: &VmxMsrs) {
|
||||
error!("MSR {name}:");
|
||||
for (idx, msr) in msr.val.iter().enumerate() {
|
||||
@@ -2244,6 +2382,7 @@ impl Vmx {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[allow(dead_code)]
|
||||
pub fn vmread(&self, field: u32) -> u64 {
|
||||
VmxAsm::vmx_vmread(field)
|
||||
}
|
||||
@@ -3006,15 +3145,10 @@ impl Vmx {
|
||||
) -> Result<i32, SystemError> {
|
||||
let exit_reason = vcpu.vmx().exit_reason;
|
||||
// self.dump_vmcs(vcpu);
|
||||
{
|
||||
let reason = self.vmread(ro::EXIT_REASON);
|
||||
debug!("vm_exit reason 0x{:x}\n", reason);
|
||||
}
|
||||
debug!("vm_exit reason {:?}\n", exit_reason);
|
||||
let unexpected_vmexit = |vcpu: &mut VirtCpu| -> Result<i32, SystemError> {
|
||||
error!("vmx: unexpected exit reason {:?}\n", exit_reason);
|
||||
|
||||
self.dump_vmcs(vcpu);
|
||||
|
||||
let cpu = vcpu.arch.last_vmentry_cpu.into() as u64;
|
||||
let run = vcpu.kvm_run_mut();
|
||||
run.exit_reason = kvm_exit::KVM_EXIT_INTERNAL_ERROR;
|
||||
@@ -3047,12 +3181,10 @@ impl Vmx {
|
||||
}
|
||||
|
||||
if exit_reason.failed_vmentry() {
|
||||
self.dump_vmcs(vcpu);
|
||||
todo!()
|
||||
}
|
||||
|
||||
if unlikely(vcpu.vmx().fail != 0) {
|
||||
self.dump_vmcs(vcpu);
|
||||
todo!()
|
||||
}
|
||||
|
||||
@@ -3272,6 +3404,7 @@ pub struct VmxVCpuPriv {
|
||||
guest_state_loaded: bool,
|
||||
|
||||
exit_qualification: u64, //暂时不知道用处fztodo
|
||||
exit_gpa: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -3333,6 +3466,7 @@ impl VmxVCpuPriv {
|
||||
exit_intr_info: IntrInfo::empty(),
|
||||
msr_autostore: VmxMsrs::default(),
|
||||
exit_qualification: 0, //fztodo
|
||||
exit_gpa: 0,
|
||||
};
|
||||
|
||||
vmx.vpid = vmx_info().alloc_vpid().unwrap_or_default() as u16;
|
||||
@@ -3637,6 +3771,11 @@ impl VmxVCpuPriv {
|
||||
pub fn get_exit_qual(&self) -> u64 {
|
||||
self.exit_qualification
|
||||
}
|
||||
|
||||
pub fn get_exit_gpa(&self) -> u64 {
|
||||
self.exit_gpa
|
||||
}
|
||||
|
||||
pub fn vmread_exit_qual(&mut self) {
|
||||
self.exit_qualification = VmxAsm::vmx_vmread(ro::EXIT_QUALIFICATION);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use core::intrinsics::unlikely;
|
||||
use core::{intrinsics::unlikely, sync::atomic::Ordering};
|
||||
|
||||
use alloc::{boxed::Box, collections::LinkedList, sync::Arc};
|
||||
use bitmap::{traits::BitMapOps, AllocBitmap};
|
||||
@@ -18,7 +18,7 @@ use crate::{
|
||||
},
|
||||
libs::spinlock::{SpinLock, SpinLockGuard},
|
||||
mm::{percpu::PerCpuVar, MemoryManagementArch, PhysAddr, VirtAddr},
|
||||
smp::cpu::ProcessorId,
|
||||
smp::cpu::{AtomicProcessorId, ProcessorId},
|
||||
};
|
||||
|
||||
use super::vmx_info;
|
||||
@@ -204,7 +204,6 @@ pub struct VmcsControlsShadow {
|
||||
pub struct LoadedVmcs {
|
||||
pub vmcs: Arc<LockedVMControlStructure>,
|
||||
pub shadow_vmcs: Option<Arc<LockedVMControlStructure>>,
|
||||
pub cpu: ProcessorId,
|
||||
/// 是否已经执行了 VMLAUNCH 指令
|
||||
pub launched: bool,
|
||||
/// NMI 是否已知未被屏蔽
|
||||
@@ -305,6 +304,7 @@ impl LoadedVmcs {
|
||||
#[derive(Debug)]
|
||||
pub struct LockedLoadedVmcs {
|
||||
inner: SpinLock<LoadedVmcs>,
|
||||
owner_cpu: AtomicProcessorId,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -334,7 +334,6 @@ impl LockedLoadedVmcs {
|
||||
inner: SpinLock::new(LoadedVmcs {
|
||||
vmcs,
|
||||
shadow_vmcs: None,
|
||||
cpu: ProcessorId::INVALID,
|
||||
launched: false,
|
||||
hv_timer_soft_disabled: false,
|
||||
msr_bitmap: bitmap,
|
||||
@@ -345,12 +344,23 @@ impl LockedLoadedVmcs {
|
||||
entry_time: 0,
|
||||
vnmi_blocked_time: 0,
|
||||
}),
|
||||
owner_cpu: AtomicProcessorId::new(ProcessorId::INVALID),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn lock(&self) -> SpinLockGuard<'_, LoadedVmcs> {
|
||||
self.inner.lock()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn owner_cpu(&self) -> ProcessorId {
|
||||
self.owner_cpu.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set_owner_cpu(&self, cpu: ProcessorId) {
|
||||
self.owner_cpu.store(cpu, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
Reference in New Issue
Block a user