Avoid frame zeroing when followed by frame copy.

rmm/src/allocator/frame/mod.rs

@@ -44,6 +44,7 @@ impl FrameUsage {
 }
 
 pub unsafe trait FrameAllocator {
+    // TODO: must_be_zero
     fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress>;
 
     unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount);

src/acpi/madt/arch/x86.rs

@@ -74,8 +74,9 @@ pub(super) fn init(madt: Madt) {
                 let cpu_id = LogicalCpuId::next();
 
                 // Allocate a stack
+                let must_be_zero = true;
                 let stack_start = RmmA::phys_to_virt(
-                    allocate_p2frame(4)
+                    allocate_p2frame(4, must_be_zero)
                         .expect("no more frames in acpi stack_start")
                         .base(),
                 )

src/arch/riscv64/misc.rs

@@ -30,7 +30,9 @@ impl PercpuBlock {
 #[cold]
 pub unsafe fn init(cpu_id: LogicalCpuId) {
     unsafe {
-        let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
+        let must_be_zero = true;
+        let frame =
+            crate::memory::allocate_frame(must_be_zero).expect("failed to allocate percpu memory");
         let virt = RmmA::phys_to_virt(frame.base()).data() as *mut ArchPercpu;
 
         virt.write(ArchPercpu {

src/arch/x86_shared/device/tsc.rs

@@ -134,7 +134,8 @@ pub unsafe fn init() -> bool {
                 .supp_feats
                 .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE)
         {
-            let frame = allocate_frame().expect("failed to allocate timer page");
+            let must_be_zero = true;
+            let frame = allocate_frame(must_be_zero).expect("failed to allocate timer page");
             x86::msr::wrmsr(MSR_KVM_SYSTEM_TIME_NEW, (frame.base().data() as u64) | 1);
             let ptr = crate::memory::RmmA::phys_to_virt(frame.base()).data()
                 as *const PvclockVcpuTimeInfo;

src/arch/x86_shared/gdt.rs

@@ -375,7 +375,9 @@ pub fn allocate_and_init_pcr(
         .next_power_of_two()
         .trailing_zeros();
 
-    let pcr_frame = crate::memory::allocate_p2frame(alloc_order).expect("failed to allocate PCR");
+    let must_be_zero = true;
+    let pcr_frame =
+        crate::memory::allocate_p2frame(alloc_order, must_be_zero).expect("failed to allocate PCR");
     let pcr_ptr = RmmA::phys_to_virt(pcr_frame.base()).data() as *mut ProcessorControlRegion;
     unsafe { core::ptr::write(pcr_ptr, ProcessorControlRegion::new_partial_init(cpu_id)) };
 

src/arch/x86_shared/idt.rs

@@ -161,7 +161,8 @@ pub fn allocate_and_init_idt(cpu_id: LogicalCpuId) -> *mut Idt {
         .or_insert_with(|| Box::leak(Box::new(Idt::new())));
 
     use crate::memory::{RmmA, RmmArch};
-    let frames = crate::memory::allocate_p2frame(4)
+    let must_be_zero = true;
+    let frames = crate::memory::allocate_p2frame(4, must_be_zero)
         .expect("failed to allocate pages for backup interrupt stack");
 
     // Physical pages are mapped linearly. So is the linearly mapped virtual memory.

src/context/context.rs

@@ -555,8 +555,9 @@ pub struct Kstack {
 }
 impl Kstack {
     pub fn new() -> Result<Self, Enomem> {
+        let must_be_zero = true;
         Ok(Self {
-            base: allocate_p2frame(4).ok_or(Enomem)?,
+            base: allocate_p2frame(4, must_be_zero).ok_or(Enomem)?,
         })
     }
     pub fn initial_top(&self) -> *mut u8 {

src/context/memory.rs

@@ -1804,7 +1804,9 @@ impl Grant {
                         Frame::containing(phys)
                     } else {
                         // TODO: Omit the unnecessary subsequent add_ref call.
-                        let new_frame = init_frame(RefCount::One).expect("TODO: handle OOM");
+                        let must_be_zero = true;
+                        let new_frame =
+                            init_frame(RefCount::One, must_be_zero).expect("TODO: handle OOM");
                         let src_flush = unsafe {
                             src_mapper
                                 .map_phys(src_page.start_address(), new_frame.base(), flags)
@@ -2482,13 +2484,19 @@ fn cow(
         });
     }
 
-    let new_frame = init_frame(initial_rc)?;
+    let must_be_zero = false;
+    let new_frame;
 
-    if old_frame != the_zeroed_frame().0 {
+    if old_frame == the_zeroed_frame().0 {
+        let must_be_zero = true;
+        new_frame = init_frame(initial_rc, must_be_zero)?;
+    } else {
+        let must_be_zero = false;
+        new_frame = init_frame(initial_rc, must_be_zero)?;
         unsafe {
             copy_frame_to_frame_directly(new_frame, old_frame);
         }
-    }
+    };
 
     Ok(CowResult {
         new_frame,
@@ -2502,7 +2510,8 @@ fn map_zeroed(
     page_flags: PageFlags<RmmA>,
     _writable: bool,
 ) -> Result<Frame, PfError> {
-    let new_frame = init_frame(RefCount::One)?;
+    let must_be_zero = true;
+    let new_frame = init_frame(RefCount::One, must_be_zero)?;
 
     unsafe {
         mapper

src/memory/mod.rs

@@ -71,17 +71,24 @@ pub fn total_frames() -> usize {
     sections().iter().map(|section| section.frames.len()).sum()
 }
 
-pub fn allocate_p2frame_with_mask(mask: FreeListMask, order: u32, fallback: bool) -> Option<Frame> {
+pub fn allocate_p2frame_with_mask(
+    mask: FreeListMask,
+    order: u32,
+    fallback: bool,
+    must_be_zero: bool,
+) -> Option<Frame> {
     let numreg = numa::number_of_memory_regions();
     if numreg == 0 {
-        return allocate_p2frame_complex(order, (), None, order, 0).map(|e| e.0);
+        return allocate_p2frame_complex(order, (), None, order, 0, must_be_zero).map(|e| e.0);
     }
 
     for i in 0..numreg {
         if mask.is_enabled(i)
             && let Some(_) = FREE_LISTS.get().unwrap().get(i)
         {
-            if let Some((frame, _)) = allocate_p2frame_complex(order, (), None, order, i) {
+            if let Some((frame, _)) =
+                allocate_p2frame_complex(order, (), None, order, i, must_be_zero)
+            {
                 return Some(frame);
             }
         }
@@ -92,7 +99,7 @@ pub fn allocate_p2frame_with_mask(mask: FreeListMask, order: u32, fallback: bool
         // from nodes in the increasing order of distance from current node
         if let Some(masks) = numa::free_lists_masks() {
             for mask in masks {
-                if let Some(frame) = allocate_p2frame_with_mask(mask, order, false) {
+                if let Some(frame) = allocate_p2frame_with_mask(mask, order, false, must_be_zero) {
                     return Some(frame);
                 }
             }
@@ -103,7 +110,9 @@ pub fn allocate_p2frame_with_mask(mask: FreeListMask, order: u32, fallback: bool
                 if !mask.is_enabled(i)
                     && let Some(_) = FREE_LISTS.get().unwrap().get(i)
                 {
-                    if let Some((frame, _)) = allocate_p2frame_complex(order, (), None, order, i) {
+                    if let Some((frame, _)) =
+                        allocate_p2frame_complex(order, (), None, order, i, must_be_zero)
+                    {
                         return Some(frame);
                     }
                 }
@@ -115,31 +124,31 @@ pub fn allocate_p2frame_with_mask(mask: FreeListMask, order: u32, fallback: bool
 }
 
 /// Allocate a range of frames
-pub fn allocate_p2frame(order: u32) -> Option<Frame> {
+pub fn allocate_p2frame(order: u32, must_be_zero: bool) -> Option<Frame> {
     static RR_INDEX: AtomicU8 = AtomicU8::new(0);
     let len = FREE_LISTS.get().unwrap().len();
 
     if len == 1 {
-        return allocate_p2frame_complex(order, (), None, order, 0).map(|e| e.0);
+        return allocate_p2frame_complex(order, (), None, order, 0, must_be_zero).map(|e| e.0);
     }
 
     // relaxed ordering since we only want atomicity
     let index =
         usize::from(RR_INDEX.fetch_add(1, Ordering::Relaxed)) % numa::number_of_memory_regions();
     for i in index..len {
-        if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i) {
+        if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i, must_be_zero) {
             return Some(frame.0);
         }
     }
     for i in 0..index {
-        if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i) {
+        if let Some(frame) = allocate_p2frame_complex(order, (), None, order, i, must_be_zero) {
             return Some(frame.0);
         }
     }
     None
 }
-pub fn allocate_frame() -> Option<Frame> {
-    allocate_p2frame(0)
+pub fn allocate_frame(must_be_zero: bool) -> Option<Frame> {
+    allocate_p2frame(0, must_be_zero)
 }
 
 // TODO: Flags, strategy
@@ -149,6 +158,7 @@ pub fn allocate_p2frame_complex(
     _strategy: Option<()>,
     min_order: u32,
     index: usize,
+    must_be_zero: bool,
 ) -> Option<(Frame, usize)> {
     let mut freelist = FREE_LISTS.get().unwrap()[index].free_list_inner.lock();
 
@@ -214,8 +224,11 @@ pub fn allocate_p2frame_complex(
     info.mark_used();
     drop(freelist);
 
+    if must_be_zero {
         unsafe {
-        (RmmA::phys_to_virt(frame.base()).data() as *mut u8).write_bytes(0, PAGE_SIZE << min_order);
+            (RmmA::phys_to_virt(frame.base()).data() as *mut u8)
+                .write_bytes(0, PAGE_SIZE << min_order);
+        }
     }
 
     debug_assert!(frame.base().data() >= unsafe { ALLOCATOR_DATA.abs_off });
@@ -447,7 +460,9 @@ pub struct RaiiFrame {
 impl RaiiFrame {
     #[cfg(not(test))]
     pub fn allocate() -> Result<Self, Enomem> {
-        init_frame(RefCount::One)
+        let must_be_zero = true;
+
+        init_frame(RefCount::One, must_be_zero)
             .map_err(|_| Enomem)
             .map(|inner| Self { inner })
     }
@@ -1020,7 +1035,11 @@ pub fn init_mm(allocator: &mut BumpAllocator<RmmA>) {
     init_sections(allocator);
 
     unsafe {
-        let the_frame = allocate_frame().expect("failed to allocate static zeroed frame");
+        // would have been interesting otherwise
+        let must_be_zero = true;
+
+        let the_frame =
+            allocate_frame(must_be_zero).expect("failed to allocate static zeroed frame");
         let the_info = get_page_info(the_frame).expect("static zeroed frame had no PageInfo");
         the_info
             .refcount
@@ -1319,8 +1338,8 @@ pub fn the_zeroed_frame() -> (Frame, &'static PageInfo) {
     }
 }
 
-pub fn init_frame(init_rc: RefCount) -> Result<Frame, PfError> {
-    let new_frame = allocate_frame().ok_or(PfError::Oom)?;
+pub fn init_frame(init_rc: RefCount, must_be_zero: bool) -> Result<Frame, PfError> {
+    let new_frame = allocate_frame(must_be_zero).ok_or(PfError::Oom)?;
     let page_info = get_page_info(new_frame).unwrap_or_else(|| {
         panic!(
             "all allocated frames need an associated page info, {:?} didn't",
@@ -1339,11 +1358,12 @@ pub struct TheFrameAllocator(pub NumaMemoryPolicy);
 
 unsafe impl FrameAllocator for TheFrameAllocator {
     fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress> {
+        let must_be_zero = true;
         let order = count.data().next_power_of_two().trailing_zeros();
         if let Some((mask, fallback)) = numa::free_list_mask(self.0) {
-            allocate_p2frame_with_mask(mask, order, fallback).map(|f| f.base())
+            allocate_p2frame_with_mask(mask, order, fallback, true).map(|f| f.base())
         } else {
-            allocate_p2frame(order).map(|f| f.base())
+            allocate_p2frame(order, true).map(|f| f.base())
         }
     }