From 7d21144da6140cda8bcc19fdd2868d361e24242e Mon Sep 17 00:00:00 2001 From: Zhang Junyang Date: Mon, 25 Aug 2025 20:49:28 +0800 Subject: [PATCH] Add RISC-V IPI --- ostd/src/arch/loongarch/irq/ipi.rs | 4 +- ostd/src/arch/riscv/irq/chip/mod.rs | 22 +++++++++-- ostd/src/arch/riscv/irq/ipi.rs | 57 +++++++++++++++++++++++++++-- ostd/src/arch/riscv/irq/mod.rs | 8 +++- ostd/src/arch/riscv/mod.rs | 23 +++++++++++- ostd/src/arch/riscv/trap/mod.rs | 9 ++++- ostd/src/arch/x86/irq/ipi.rs | 22 +++++++++-- ostd/src/arch/x86/irq/mod.rs | 2 +- ostd/src/arch/x86/mod.rs | 1 + ostd/src/smp.rs | 31 ++++++---------- 10 files changed, 143 insertions(+), 36 deletions(-) diff --git a/ostd/src/arch/loongarch/irq/ipi.rs b/ostd/src/arch/loongarch/irq/ipi.rs index 68115ce96..a53b025c9 100644 --- a/ostd/src/arch/loongarch/irq/ipi.rs +++ b/ostd/src/arch/loongarch/irq/ipi.rs @@ -22,6 +22,8 @@ impl HwCpuId { /// The caller must ensure that the interrupt number is valid and that /// the corresponding handler is configured correctly on the remote CPU. /// Furthermore, invoking the interrupt handler must also be safe. -pub(crate) unsafe fn send_ipi(_hw_cpu_id: HwCpuId, _irq_num: u8, _guard: &dyn PinCurrentCpu) { +pub(crate) unsafe fn send_ipi(_hw_cpu_id: HwCpuId, _guard: &dyn PinCurrentCpu) { + // To suppress unused function lint errors. We should be using it here. + let _ = crate::smp::do_inter_processor_call; unimplemented!() } diff --git a/ostd/src/arch/riscv/irq/chip/mod.rs b/ostd/src/arch/riscv/irq/chip/mod.rs index 471338c64..9d1a8b4c4 100644 --- a/ostd/src/arch/riscv/irq/chip/mod.rs +++ b/ostd/src/arch/riscv/irq/chip/mod.rs @@ -41,10 +41,24 @@ pub(in crate::arch) unsafe fn init(io_mem_builder: &IoMemAllocatorBuilder) { IRQ_CHIP.call_once(|| IrqChip { plics: SpinLock::new(plics.into_boxed_slice()), }); - // SAFETY: Accessing the `sie` CSR to enable the external interrupt is - // safe here because this function is only called during PLIC - // initialization, and we ensure that only the external interrupt bit is - // set without affecting other interrupt sources. + // SAFETY: Accessing the `sie` CSR to enable the external interrupt is safe + // here because this function is only called during PLIC initialization, + // and we ensure that only the external interrupt bit is set without + // affecting other interrupt sources. + unsafe { riscv::register::sie::set_sext() }; +} + +/// Initializes application-processor-specific PLIC state. +/// +/// # Safety +/// +/// This function is safe to call on the following conditions: +/// 1. It is called once and at most once on this AP. +/// 2. It is called before any other public functions of this module is called +/// on this AP. +pub(in crate::arch) unsafe fn init_current_hart() { + // SAFETY: Accessing the `sie` CSR to enable the external interrupt is safe + // here due to the same reasons mentioned in `init`. unsafe { riscv::register::sie::set_sext() }; } diff --git a/ostd/src/arch/riscv/irq/ipi.rs b/ostd/src/arch/riscv/irq/ipi.rs index c32c174fe..d6ff2f898 100644 --- a/ostd/src/arch/riscv/irq/ipi.rs +++ b/ostd/src/arch/riscv/irq/ipi.rs @@ -2,7 +2,9 @@ //! Inter-processor interrupts. -use crate::cpu::PinCurrentCpu; +use spin::Once; + +use crate::{cpu::PinCurrentCpu, irq::IrqLine}; /// Hardware-specific, architecture-dependent CPU ID. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -15,6 +17,37 @@ impl HwCpuId { } } +pub(in crate::arch) static IPI_IRQ: Once = Once::new(); + +/// Initializes the global IPI-related state and local state on BSP. +/// +/// # Safety +/// +/// This function can only be called before any other IPI-related function is +/// called. +pub(in crate::arch) unsafe fn init() { + let mut irq = IrqLine::alloc().unwrap(); + // SAFETY: This will be called upon an inter-processor interrupt. + irq.on_active(|f| unsafe { crate::smp::do_inter_processor_call(f) }); + IPI_IRQ.call_once(|| irq); + + // SAFETY: Enabling the software interrupts is safe here because this + // function cannot be called when others can perform IPI-related + // operations. And it has no side-effects. + unsafe { riscv::register::sie::set_ssoft() }; +} + +/// Initializes the IPI-related state on this CPU. +/// +/// # Safety +/// +/// This function can only be called before any other harts can IPI this hart. +pub(in crate::arch) unsafe fn init_current_hart() { + // SAFETY: Enabling the software interrupts is safe here due to the same + // reasons mentioned in `init`. + unsafe { riscv::register::sie::set_ssoft() }; +} + /// Sends a general inter-processor interrupt (IPI) to the specified CPU. /// /// # Safety @@ -22,6 +55,24 @@ impl HwCpuId { /// The caller must ensure that the interrupt number is valid and that /// the corresponding handler is configured correctly on the remote CPU. /// Furthermore, invoking the interrupt handler must also be safe. -pub(crate) unsafe fn send_ipi(_hw_cpu_id: HwCpuId, _irq_num: u8, _guard: &dyn PinCurrentCpu) { - unimplemented!() +#[expect(unused_variables)] +pub(crate) unsafe fn send_ipi(hw_cpu_id: HwCpuId, guard: &dyn PinCurrentCpu) { + const XLEN: usize = core::mem::size_of::() * 8; + const XLEN_MASK: usize = XLEN - 1; + + let hart_id = hw_cpu_id.0 as usize; + let hart_mask_base = hart_id & !XLEN_MASK; + let hart_mask = 1 << (hart_id & XLEN_MASK); + + let ret = sbi_rt::send_ipi(sbi_rt::HartMask::from_mask_base(hart_mask, hart_mask_base)); + + if ret.error == 0 { + log::debug!("Successfully sent IPI to hart {}", hw_cpu_id.0); + } else { + log::error!( + "Failed to send IPI to hart {}: error code {}", + hw_cpu_id.0, + ret.error + ); + } } diff --git a/ostd/src/arch/riscv/irq/mod.rs b/ostd/src/arch/riscv/irq/mod.rs index f27d803ea..fa9e9bd8b 100644 --- a/ostd/src/arch/riscv/irq/mod.rs +++ b/ostd/src/arch/riscv/irq/mod.rs @@ -3,7 +3,7 @@ //! Interrupts. pub(super) mod chip; -mod ipi; +pub(super) mod ipi; mod ops; mod remapping; @@ -55,7 +55,11 @@ impl HwIrqLine { *interrupt_source_on_chip, ); } - InterruptSource::Software => unimplemented!(), + InterruptSource::Software => { + // SAFETY: We have already handled the IPI. So clearing the + // software interrupt pending bit is safe. + unsafe { riscv::register::sip::clear_ssoft() }; + } } } } diff --git a/ostd/src/arch/riscv/mod.rs b/ostd/src/arch/riscv/mod.rs index 6725a81f9..cddeae567 100644 --- a/ostd/src/arch/riscv/mod.rs +++ b/ostd/src/arch/riscv/mod.rs @@ -45,6 +45,9 @@ pub(crate) unsafe fn late_init_on_bsp() { // operations having been performed. unsafe { irq::chip::init(&io_mem_builder) }; + // SAFETY: This is called before any IPI-related operation is performed. + unsafe { irq::ipi::init() }; + // SAFETY: We're on the BSP and we're ready to boot all APs. unsafe { crate::boot::smp::boot_all_aps() }; @@ -59,8 +62,26 @@ pub(crate) unsafe fn late_init_on_bsp() { unsafe { crate::io::init(io_mem_builder) }; } +/// Initializes application-processor-specific state. +/// +/// On RISC-V, Application Processors (APs) are harts that are not the +/// bootstrapping hart. +/// +/// # Safety +/// +/// This function must be called only once on each application processor. +/// And it should be called after the BSP's call to [`late_init_on_bsp`] +/// and before any other architecture-specific code in this module is called on +/// this AP. pub(crate) unsafe fn init_on_ap() { - unimplemented!(); + // SAFETY: The safety is upheld by the caller. + unsafe { trap::init_on_cpu() }; + + // SAFETY: The safety is upheld by the caller. + unsafe { irq::chip::init_current_hart() }; + + // SAFETY: This is called before any IPI-related operation is performed. + unsafe { irq::ipi::init_current_hart() }; } /// Returns the frequency of TSC. The unit is Hz. diff --git a/ostd/src/arch/riscv/trap/mod.rs b/ostd/src/arch/riscv/trap/mod.rs index c59229420..e5d8233fa 100644 --- a/ostd/src/arch/riscv/trap/mod.rs +++ b/ostd/src/arch/riscv/trap/mod.rs @@ -127,7 +127,14 @@ pub(super) fn call_irq_callback_functions_by_scause( call_irq_callback_functions(trap_frame, &hw_irq_line, priv_level); } } - Interrupt::SupervisorSoft => todo!(), + Interrupt::SupervisorSoft => { + let ipi_irq_num = super::irq::ipi::IPI_IRQ.get().unwrap().num(); + call_irq_callback_functions( + trap_frame, + &HwIrqLine::new(ipi_irq_num, InterruptSource::Software), + priv_level, + ); + } Interrupt::Unknown => { panic!( "Cannot handle unknown supervisor interrupt, scause: {:#x}, trapframe: {:#x?}.", diff --git a/ostd/src/arch/x86/irq/ipi.rs b/ostd/src/arch/x86/irq/ipi.rs index 638c49ac4..b5abaa1db 100644 --- a/ostd/src/arch/x86/irq/ipi.rs +++ b/ostd/src/arch/x86/irq/ipi.rs @@ -2,7 +2,9 @@ //! Inter-processor interrupts. -use crate::cpu::PinCurrentCpu; +use spin::Once; + +use crate::{cpu::PinCurrentCpu, irq::IrqLine, smp::do_inter_processor_call}; /// Hardware-specific, architecture-dependent CPU ID. /// @@ -19,6 +21,16 @@ impl HwCpuId { } } +static IPI_IRQ: Once = Once::new(); + +/// Initializes global IPI state. +pub(in crate::arch) fn init() { + let mut irq = IrqLine::alloc().unwrap(); + // SAFETY: This will be called upon an inter-processor interrupt. + irq.on_active(|f| unsafe { do_inter_processor_call(f) }); + IPI_IRQ.call_once(|| irq); +} + /// Sends a general inter-processor interrupt (IPI) to the specified CPU. /// /// # Safety @@ -26,9 +38,11 @@ impl HwCpuId { /// The caller must ensure that the interrupt number is valid and that /// the corresponding handler is configured correctly on the remote CPU. /// Furthermore, invoking the interrupt handler must also be safe. -pub(crate) unsafe fn send_ipi(hw_cpu_id: HwCpuId, irq_num: u8, guard: &dyn PinCurrentCpu) { +pub(crate) unsafe fn send_ipi(hw_cpu_id: HwCpuId, guard: &dyn PinCurrentCpu) { use crate::arch::kernel::apic::{self, Icr}; + let irq_num = IPI_IRQ.get().unwrap().num(); + let icr = Icr::new( apic::ApicId::from(hw_cpu_id.0), apic::DestinationShorthand::NoShorthand, @@ -41,7 +55,7 @@ pub(crate) unsafe fn send_ipi(hw_cpu_id: HwCpuId, irq_num: u8, guard: &dyn PinCu ); let apic = apic::get_or_init(guard); - // SAFETY: The ICR is valid to generate the request IPI. Generating the request IPI is safe - // as guaranteed by the caller. + // SAFETY: The ICR is valid to generate the request IPI. Generating the + // request IPI is safe. unsafe { apic.send_ipi(icr) }; } diff --git a/ostd/src/arch/x86/irq/mod.rs b/ostd/src/arch/x86/irq/mod.rs index f08b2c621..25571f533 100644 --- a/ostd/src/arch/x86/irq/mod.rs +++ b/ostd/src/arch/x86/irq/mod.rs @@ -3,7 +3,7 @@ //! Interrupts. pub(super) mod chip; -mod ipi; +pub(super) mod ipi; mod ops; mod remapping; diff --git a/ostd/src/arch/x86/mod.rs b/ostd/src/arch/x86/mod.rs index c9159f906..a343e6700 100644 --- a/ostd/src/arch/x86/mod.rs +++ b/ostd/src/arch/x86/mod.rs @@ -60,6 +60,7 @@ pub(crate) unsafe fn late_init_on_bsp() { kernel::apic::init(&io_mem_builder).expect("APIC doesn't exist"); irq::chip::init(&io_mem_builder); + irq::ipi::init(); kernel::tsc::init_tsc_freq(); timer::init_on_bsp(); diff --git a/ostd/src/smp.rs b/ostd/src/smp.rs index 250c29444..11a7dfa6c 100644 --- a/ostd/src/smp.rs +++ b/ostd/src/smp.rs @@ -10,13 +10,9 @@ use alloc::{boxed::Box, collections::VecDeque}; use spin::Once; use crate::{ - arch::{ - irq::{send_ipi, HwCpuId}, - trap::TrapFrame, - }, + arch::{irq::HwCpuId, trap::TrapFrame}, cpu::{CpuSet, PinCurrentCpu}, - cpu_local, - irq::{self, IrqLine}, + cpu_local, irq, sync::SpinLock, util::id_set::Id, }; @@ -40,7 +36,6 @@ pub fn inter_processor_call(targets: &CpuSet, f: fn()) { let this_cpu_id = irq_guard.current_cpu(); let ipi_data = IPI_GLOBAL_DATA.get().unwrap(); - let irq_num = ipi_data.irq.num(); let mut call_on_self = false; for cpu_id in targets.iter() { @@ -54,15 +49,12 @@ pub fn inter_processor_call(targets: &CpuSet, f: fn()) { if cpu_id == this_cpu_id { continue; } + let hw_cpu_id = ipi_data.hw_cpu_ids[cpu_id.as_usize()]; // SAFETY: The value of `irq_num` corresponds to a valid IRQ line and // triggering it will not cause any safety issues. unsafe { - send_ipi( - ipi_data.hw_cpu_ids[cpu_id.as_usize()], - irq_num, - &irq_guard as _, - ) - }; + crate::arch::irq::send_ipi(hw_cpu_id, &irq_guard as _); + } } if call_on_self { // Execute the function synchronously. @@ -71,7 +63,6 @@ pub fn inter_processor_call(targets: &CpuSet, f: fn()) { } struct IpiGlobalData { - irq: IrqLine, hw_cpu_ids: Box<[HwCpuId]>, } @@ -81,7 +72,12 @@ cpu_local! { static CALL_QUEUES: SpinLock> = SpinLock::new(VecDeque::new()); } -fn do_inter_processor_call(_trapframe: &TrapFrame) { +/// Handles inter-processor calls. +/// +/// # Safety +/// +/// It should only be called upon the inter processor interrupt. +pub(crate) unsafe fn do_inter_processor_call(_trapframe: &TrapFrame) { // No races because we are in IRQs. let this_cpu_id = crate::cpu::CpuId::current_racy(); @@ -98,11 +94,8 @@ fn do_inter_processor_call(_trapframe: &TrapFrame) { pub(super) fn init() { IPI_GLOBAL_DATA.call_once(|| { - let mut irq = IrqLine::alloc().unwrap(); - irq.on_active(do_inter_processor_call); - let hw_cpu_ids = crate::boot::smp::construct_hw_cpu_id_mapping(); - IpiGlobalData { irq, hw_cpu_ids } + IpiGlobalData { hw_cpu_ids } }); }