From 4b7365d53e0ba905a412e5ad06015a9d5b1660d2 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:14:07 -0700 Subject: [PATCH 01/67] powerpc/pseries/iommu: Fix kmemleak in TCE table userspace view JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit d36e3f11fe8b55b801bdbe84ad51f612b1bd84da Author: Gaurav Batra Date: Mon May 12 17:46:53 2025 -0500 powerpc/pseries/iommu: Fix kmemleak in TCE table userspace view When a device is opened by a userspace driver, via VFIO interface, DMA window is created. This DMA window has TCE Table and a corresponding data for userview of TCE table. When the userspace driver closes the device, all the above infrastructure is free'ed and the device control given back to kernel. Both DMA window and TCE table is getting free'ed. But due to a code bug, userview of the TCE table is not getting free'ed. This is resulting in a memory leak. Befow is the information from KMEMLEAK unreferenced object 0xc008000022af0000 (size 16777216): comm "senlib_unit_tes", pid 9346, jiffies 4294983174 hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace (crc 0): kmemleak_vmalloc+0xc8/0x1a0 __vmalloc_node_range+0x284/0x340 vzalloc+0x58/0x70 spapr_tce_create_table+0x4b0/0x8d0 tce_iommu_create_table+0xcc/0x170 [vfio_iommu_spapr_tce] tce_iommu_create_window+0x144/0x2f0 [vfio_iommu_spapr_tce] tce_iommu_ioctl.part.0+0x59c/0xc90 [vfio_iommu_spapr_tce] vfio_fops_unl_ioctl+0x88/0x280 [vfio] sys_ioctl+0xf4/0x160 system_call_exception+0x164/0x310 system_call_vectored_common+0xe8/0x278 unreferenced object 0xc008000023b00000 (size 4194304): comm "senlib_unit_tes", pid 9351, jiffies 4294984116 hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace (crc 0): kmemleak_vmalloc+0xc8/0x1a0 __vmalloc_node_range+0x284/0x340 vzalloc+0x58/0x70 spapr_tce_create_table+0x4b0/0x8d0 tce_iommu_create_table+0xcc/0x170 [vfio_iommu_spapr_tce] tce_iommu_create_window+0x144/0x2f0 [vfio_iommu_spapr_tce] tce_iommu_create_default_window+0x88/0x120 [vfio_iommu_spapr_tce] tce_iommu_ioctl.part.0+0x57c/0xc90 [vfio_iommu_spapr_tce] vfio_fops_unl_ioctl+0x88/0x280 [vfio] sys_ioctl+0xf4/0x160 system_call_exception+0x164/0x310 system_call_vectored_common+0xe8/0x278 Fixes: f431a8cde7f1 ("powerpc/iommu: Reimplement the iommu_table_group_ops for pSeries") Signed-off-by: Gaurav Batra Reviewed-by: Nilay Shroff Reviewed-by: Ritesh Harjani (IBM) Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20250512224653.35697-1-gbatra@linux.ibm.com (cherry picked from commit d36e3f11fe8b55b801bdbe84ad51f612b1bd84da) Signed-off-by: Jerry Snitselaar --- arch/powerpc/platforms/pseries/iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index e3e803debfe3..3dc61bb34a8a 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -198,7 +198,7 @@ static void tce_iommu_userspace_view_free(struct iommu_table *tbl) static void tce_free_pSeries(struct iommu_table *tbl) { - if (!tbl->it_userspace) + if (tbl->it_userspace) tce_iommu_userspace_view_free(tbl); } From 2cf9724afc30056d60e7b639b068cbe3586d9e76 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:14:48 -0700 Subject: [PATCH 02/67] scatterlist: fix incorrect func name in kernel-doc JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit d89c8ec0546184267cb211b579514ebaf8916100 Author: Randy Dunlap Date: Fri Nov 29 18:24:06 2024 -0800 scatterlist: fix incorrect func name in kernel-doc Fix a kernel-doc warning by making the kernel-doc function description match the function name: include/linux/scatterlist.h:323: warning: expecting prototype for sg_unmark_bus_address(). Prototype was for sg_dma_unmark_bus_address() instead Link: https://lkml.kernel.org/r/20241130022406.537973-1-rdunlap@infradead.org Fixes: 42399301203e ("lib/scatterlist: add flag for indicating P2PDMA segments in an SGL") Signed-off-by: Randy Dunlap Cc: Logan Gunthorpe Cc: Christoph Hellwig Signed-off-by: Andrew Morton (cherry picked from commit d89c8ec0546184267cb211b579514ebaf8916100) Signed-off-by: Jerry Snitselaar --- include/linux/scatterlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h index c3bdee099954..4c318eb80f87 100644 --- a/include/linux/scatterlist.h +++ b/include/linux/scatterlist.h @@ -335,7 +335,7 @@ static inline void sg_dma_mark_bus_address(struct scatterlist *sg) } /** - * sg_unmark_bus_address - Unmark the scatterlist entry as a bus address + * sg_dma_unmark_bus_address - Unmark the scatterlist entry as a bus address * @sg: SG entry * * Description: From 3ff14b13d9825a196ece4bd00cf639db59fdf128 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:15:04 -0700 Subject: [PATCH 03/67] lib: scatterlist: fix sg_split_phys to preserve original scatterlist offsets JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 8b46fdaea819a679da176b879e7b0674a1161a5e Author: T Pratham Date: Wed Mar 19 16:44:38 2025 +0530 lib: scatterlist: fix sg_split_phys to preserve original scatterlist offsets The split_sg_phys function was incorrectly setting the offsets of all scatterlist entries (except the first) to 0. Only the first scatterlist entry's offset and length needs to be modified to account for the skip. Setting the rest entries' offsets to 0 could lead to incorrect data access. I am using this function in a crypto driver that I'm currently developing (not yet sent to mailing list). During testing, it was observed that the output scatterlists (except the first one) contained incorrect garbage data. I narrowed this issue down to the call of sg_split(). Upon debugging inside this function, I found that this resetting of offset is the cause of the problem, causing the subsequent scatterlists to point to incorrect memory locations in a page. By removing this code, I am obtaining expected data in all the split output scatterlists. Thus, this was indeed causing observable runtime effects! This patch removes the offending code, ensuring that the page offsets in the input scatterlist are preserved in the output scatterlist. Link: https://lkml.kernel.org/r/20250319111437.1969903-1-t-pratham@ti.com Fixes: f8bcbe62acd0 ("lib: scatterlist: add sg splitting function") Signed-off-by: T Pratham Cc: Robert Jarzmik Cc: Jens Axboe Cc: Kamlesh Gurudasani Cc: Praneeth Bajjuri Cc: Vignesh Raghavendra Cc: Signed-off-by: Andrew Morton (cherry picked from commit 8b46fdaea819a679da176b879e7b0674a1161a5e) Signed-off-by: Jerry Snitselaar --- lib/sg_split.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/sg_split.c b/lib/sg_split.c index 60a0babebf2e..0f89aab5c671 100644 --- a/lib/sg_split.c +++ b/lib/sg_split.c @@ -88,8 +88,6 @@ static void sg_split_phys(struct sg_splitter *splitters, const int nb_splits) if (!j) { out_sg->offset += split->skip_sg0; out_sg->length -= split->skip_sg0; - } else { - out_sg->offset = 0; } sg_dma_address(out_sg) = 0; sg_dma_len(out_sg) = 0; From 021deec56900ade7f57a9a0bb526a802b6ac9e99 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:16:47 -0700 Subject: [PATCH 04/67] x86/sev: Fix broken SNP support with KVM module built-in JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 409f45387c937145adeeeebc6d6032c2ec232b35 Author: Ashish Kalra Date: Mon Feb 10 22:54:18 2025 +0000 x86/sev: Fix broken SNP support with KVM module built-in Fix issues with enabling SNP host support and effectively SNP support which is broken with respect to the KVM module being built-in. SNP host support is enabled in snp_rmptable_init() which is invoked as device_initcall(). SNP check on IOMMU is done during IOMMU PCI init (IOMMU_PCI_INIT stage). And for that reason snp_rmptable_init() is currently invoked via device_initcall() and cannot be invoked via subsys_initcall() as core IOMMU subsystem gets initialized via subsys_initcall(). Now, if kvm_amd module is built-in, it gets initialized before SNP host support is enabled in snp_rmptable_init() : [ 10.131811] kvm_amd: TSC scaling supported [ 10.136384] kvm_amd: Nested Virtualization enabled [ 10.141734] kvm_amd: Nested Paging enabled [ 10.146304] kvm_amd: LBR virtualization supported [ 10.151557] kvm_amd: SEV enabled (ASIDs 100 - 509) [ 10.156905] kvm_amd: SEV-ES enabled (ASIDs 1 - 99) [ 10.162256] kvm_amd: SEV-SNP enabled (ASIDs 1 - 99) [ 10.171508] kvm_amd: Virtual VMLOAD VMSAVE supported [ 10.177052] kvm_amd: Virtual GIF supported ... ... [ 10.201648] kvm_amd: in svm_enable_virtualization_cpu And then svm_x86_ops->enable_virtualization_cpu() (svm_enable_virtualization_cpu) programs MSR_VM_HSAVE_PA as following: wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); So VM_HSAVE_PA is non-zero before SNP support is enabled on all CPUs. snp_rmptable_init() gets invoked after svm_enable_virtualization_cpu() as following : ... [ 11.256138] kvm_amd: in svm_enable_virtualization_cpu ... [ 11.264918] SEV-SNP: in snp_rmptable_init This triggers a #GP exception in snp_rmptable_init() when snp_enable() is invoked to set SNP_EN in SYSCFG MSR: [ 11.294289] unchecked MSR access error: WRMSR to 0xc0010010 (tried to write 0x0000000003fc0000) at rIP: 0xffffffffaf5d5c28 (native_write_msr+0x8/0x30) ... [ 11.294404] Call Trace: [ 11.294482] [ 11.294513] ? show_stack_regs+0x26/0x30 [ 11.294522] ? ex_handler_msr+0x10f/0x180 [ 11.294529] ? search_extable+0x2b/0x40 [ 11.294538] ? fixup_exception+0x2dd/0x340 [ 11.294542] ? exc_general_protection+0x14f/0x440 [ 11.294550] ? asm_exc_general_protection+0x2b/0x30 [ 11.294557] ? __pfx_snp_enable+0x10/0x10 [ 11.294567] ? native_write_msr+0x8/0x30 [ 11.294570] ? __snp_enable+0x5d/0x70 [ 11.294575] snp_enable+0x19/0x20 [ 11.294578] __flush_smp_call_function_queue+0x9c/0x3a0 [ 11.294586] generic_smp_call_function_single_interrupt+0x17/0x20 [ 11.294589] __sysvec_call_function+0x20/0x90 [ 11.294596] sysvec_call_function+0x80/0xb0 [ 11.294601] [ 11.294603] [ 11.294605] asm_sysvec_call_function+0x1f/0x30 ... [ 11.294631] arch_cpu_idle+0xd/0x20 [ 11.294633] default_idle_call+0x34/0xd0 [ 11.294636] do_idle+0x1f1/0x230 [ 11.294643] ? complete+0x71/0x80 [ 11.294649] cpu_startup_entry+0x30/0x40 [ 11.294652] start_secondary+0x12d/0x160 [ 11.294655] common_startup_64+0x13e/0x141 [ 11.294662] This #GP exception is getting triggered due to the following errata for AMD family 19h Models 10h-1Fh Processors: Processor may generate spurious #GP(0) Exception on WRMSR instruction: Description: The Processor will generate a spurious #GP(0) Exception on a WRMSR instruction if the following conditions are all met: - the target of the WRMSR is a SYSCFG register. - the write changes the value of SYSCFG.SNPEn from 0 to 1. - One of the threads that share the physical core has a non-zero value in the VM_HSAVE_PA MSR. The document being referred to above: https://www.amd.com/content/dam/amd/en/documents/processor-tech-docs/revision-guides/57095-PUB_1_01.pdf To summarize, with kvm_amd module being built-in, KVM/SVM initialization happens before host SNP is enabled and this SVM initialization sets VM_HSAVE_PA to non-zero, which then triggers a #GP when SYSCFG.SNPEn is being set and this will subsequently cause SNP_INIT(_EX) to fail with INVALID_CONFIG error as SYSCFG[SnpEn] is not set on all CPUs. Essentially SNP host enabling code should be invoked before KVM initialization, which is currently not the case when KVM is built-in. Add fix to call snp_rmptable_init() early from iommu_snp_enable() directly and not invoked via device_initcall() which enables SNP host support before KVM initialization with kvm_amd module built-in. Add additional handling for `iommu=off` or `amd_iommu=off` options. Note that IOMMUs need to be enabled for SNP initialization, therefore, if host SNP support is enabled but late IOMMU initialization fails then that will cause PSP driver's SNP_INIT to fail as IOMMU SNP sanity checks in SNP firmware will fail with invalid configuration error as below: [ 9.723114] ccp 0000:23:00.1: sev enabled [ 9.727602] ccp 0000:23:00.1: psp enabled [ 9.732527] ccp 0000:a2:00.1: enabling device (0000 -> 0002) [ 9.739098] ccp 0000:a2:00.1: no command queues available [ 9.745167] ccp 0000:a2:00.1: psp enabled [ 9.805337] ccp 0000:23:00.1: SEV-SNP: failed to INIT rc -5, error 0x3 [ 9.866426] ccp 0000:23:00.1: SEV API:1.53 build:5 Fixes: c3b86e61b756 ("x86/cpufeatures: Enable/unmask SEV-SNP CPU feature") Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Cc: Signed-off-by: Ashish Kalra Acked-by: Joerg Roedel Message-ID: <138b520fb83964782303b43ade4369cd181fdd9c.1739226950.git.ashish.kalra@amd.com> Signed-off-by: Paolo Bonzini (cherry picked from commit 409f45387c937145adeeeebc6d6032c2ec232b35) Signed-off-by: Jerry Snitselaar --- arch/x86/include/asm/sev.h | 2 ++ arch/x86/virt/svm/sev.c | 23 +++++++---------------- drivers/iommu/amd/init.c | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index a9f8e97f786d..494d634e8611 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -470,6 +470,7 @@ static inline int snp_svsm_vtpm_send_command(u8 *buffer) { return -ENODEV; } #ifdef CONFIG_KVM_AMD_SEV bool snp_probe_rmptable_info(void); +int snp_rmptable_init(void); int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level); void snp_dump_hva_rmpentry(unsigned long address); int psmash(u64 pfn); @@ -484,6 +485,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages) } #else static inline bool snp_probe_rmptable_info(void) { return false; } +static inline int snp_rmptable_init(void) { return -ENOSYS; } static inline int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) { return -ENODEV; } static inline void snp_dump_hva_rmpentry(unsigned long address) {} static inline int psmash(u64 pfn) { return -ENODEV; } diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c index c1a9f1a53ca3..1bcc05540198 100644 --- a/arch/x86/virt/svm/sev.c +++ b/arch/x86/virt/svm/sev.c @@ -505,19 +505,19 @@ static bool __init setup_rmptable(void) * described in the SNP_INIT_EX firmware command description in the SNP * firmware ABI spec. */ -static int __init snp_rmptable_init(void) +int __init snp_rmptable_init(void) { unsigned int i; u64 val; - if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) - return 0; + if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP))) + return -ENOSYS; - if (!amd_iommu_snp_en) - goto nosnp; + if (WARN_ON_ONCE(!amd_iommu_snp_en)) + return -ENOSYS; if (!setup_rmptable()) - goto nosnp; + return -ENOSYS; /* * Check if SEV-SNP is already enabled, this can happen in case of @@ -530,7 +530,7 @@ static int __init snp_rmptable_init(void) /* Zero out the RMP bookkeeping area */ if (!clear_rmptable_bookkeeping()) { free_rmp_segment_table(); - goto nosnp; + return -ENOSYS; } /* Zero out the RMP entries */ @@ -562,17 +562,8 @@ skip_enable: crash_kexec_post_notifiers = true; return 0; - -nosnp: - cc_platform_clear(CC_ATTR_HOST_SEV_SNP); - return -ENOSYS; } -/* - * This must be called after the IOMMU has been initialized. - */ -device_initcall(snp_rmptable_init); - static void set_rmp_segment_info(unsigned int segment_shift) { rmp_segment_shift = segment_shift; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 7bee11ccfeec..3dd0c2ac9274 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -3365,7 +3365,7 @@ out: return true; } -static void iommu_snp_enable(void) +static __init void iommu_snp_enable(void) { #ifdef CONFIG_KVM_AMD_SEV if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) @@ -3390,6 +3390,14 @@ static void iommu_snp_enable(void) goto disable_snp; } + /* + * Enable host SNP support once SNP support is checked on IOMMU. + */ + if (snp_rmptable_init()) { + pr_warn("SNP: RMP initialization failed, SNP cannot be supported.\n"); + goto disable_snp; + } + pr_info("IOMMU SNP support enabled.\n"); return; @@ -3531,6 +3539,19 @@ static int __init iommu_go_to_state(enum iommu_init_state state) ret = state_next(); } + /* + * SNP platform initilazation requires IOMMUs to be fully configured. + * If the SNP support on IOMMUs has NOT been checked, simply mark SNP + * as unsupported. If the SNP support on IOMMUs has been checked and + * host SNP support enabled but RMP enforcement has not been enabled + * in IOMMUs, then the system is in a half-baked state, but can limp + * along as all memory should be Hypervisor-Owned in the RMP. WARN, + * but leave SNP as "supported" to avoid confusing the kernel. + */ + if (ret && cc_platform_has(CC_ATTR_HOST_SEV_SNP) && + !WARN_ON_ONCE(amd_iommu_snp_en)) + cc_platform_clear(CC_ATTR_HOST_SEV_SNP); + return ret; } @@ -3638,18 +3659,23 @@ void __init amd_iommu_detect(void) int ret; if (no_iommu || (iommu_detected && !gart_iommu_aperture)) - return; + goto disable_snp; if (!amd_iommu_sme_check()) - return; + goto disable_snp; ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); if (ret) - return; + goto disable_snp; amd_iommu_detected = true; iommu_detected = 1; x86_init.iommu.iommu_init = amd_iommu_init; + return; + +disable_snp: + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + cc_platform_clear(CC_ATTR_HOST_SEV_SNP); } /**************************************************************************** From 77b7f9abab71df96ddb1f66c82481575463866a9 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:19:02 -0700 Subject: [PATCH 05/67] dma-mapping: benchmark: Restore padding to ensure uABI remained consistent JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 23ee8a2563a0f24cf4964685ced23c32be444ab8 Author: Qinxin Xia Date: Tue Oct 28 20:08:59 2025 +0800 dma-mapping: benchmark: Restore padding to ensure uABI remained consistent The padding field in the structure was previously reserved to maintain a stable interface for potential new fields, ensuring compatibility with user-space shared data structures. However,it was accidentally removed by tiantao in a prior commit, which may lead to incompatibility between user space and the kernel. This patch reinstates the padding to restore the original structure layout and preserve compatibility. Fixes: 8ddde07a3d28 ("dma-mapping: benchmark: extract a common header file for map_benchmark definition") Cc: stable@vger.kernel.org Acked-by: Barry Song Signed-off-by: Qinxin Xia Reported-by: Barry Song Closes: https://lore.kernel.org/lkml/CAGsJ_4waiZ2+NBJG+SCnbNk+nQ_ZF13_Q5FHJqZyxyJTcEop2A@mail.gmail.com/ Reviewed-by: Jonathan Cameron Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20251028120900.2265511-2-xiaqinxin@huawei.com (cherry picked from commit 23ee8a2563a0f24cf4964685ced23c32be444ab8) Signed-off-by: Jerry Snitselaar --- include/uapi/linux/map_benchmark.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/map_benchmark.h b/include/uapi/linux/map_benchmark.h index 88986fe2a381..c2d91088a40d 100644 --- a/include/uapi/linux/map_benchmark.h +++ b/include/uapi/linux/map_benchmark.h @@ -29,6 +29,7 @@ struct map_benchmark { __u32 dma_dir; /* DMA data direction */ __u32 dma_trans_ns; /* time for DMA transmission in ns */ __u32 granule; /* how many PAGE_SIZE will do map/unmap once a time */ + __u8 expansion[76]; /* For future use */ }; #endif /* _UAPI_DMA_BENCHMARK_H */ From a8fb77004a3ef5c37f265b581db34ea7907efdae Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:21:00 -0700 Subject: [PATCH 06/67] iommufd: vfio compatibility extension check for noiommu mode JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 7147ec874ea08c322d779d8eba28946e294ed1f3 Author: Jacob Pan Date: Fri Feb 13 10:36:36 2026 -0800 iommufd: vfio compatibility extension check for noiommu mode VFIO_CHECK_EXTENSION should return false for TYPE1_IOMMU variants when in NO-IOMMU mode and IOMMUFD compat container is set. This change makes the behavior match VFIO_CONTAINER in noiommu mode. It also prevents userspace from incorrectly attempting to use TYPE1 IOMMU operations in a no-iommu context. Fixes: d624d6652a65 ("iommufd: vfio container FD ioctl compatibility") Link: https://patch.msgid.link/r/20260213183636.3340-1-jacob.pan@linux.microsoft.com Signed-off-by: Jacob Pan Signed-off-by: Jason Gunthorpe (cherry picked from commit 7147ec874ea08c322d779d8eba28946e294ed1f3) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/vfio_compat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c index a258ee2f4579..acb48cdd3b00 100644 --- a/drivers/iommu/iommufd/vfio_compat.c +++ b/drivers/iommu/iommufd/vfio_compat.c @@ -283,7 +283,7 @@ static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx, case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: case VFIO_UNMAP_ALL: - return 1; + return !ictx->no_iommu_mode; case VFIO_NOIOMMU_IOMMU: return IS_ENABLED(CONFIG_VFIO_NOIOMMU); From b6d92a65b83c8986ae6efaff9010fcad4ee19d7e Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:21:22 -0700 Subject: [PATCH 07/67] iommufd/selftest: Fix page leaks in mock_viommu_{init,destroy} JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 09c091fddb0b93297ea659ab48ee64f54ebeeaa2 Author: Thorsten Blum Date: Thu Mar 12 17:40:42 2026 +0100 iommufd/selftest: Fix page leaks in mock_viommu_{init,destroy} mock_viommu_init() allocates two pages using __get_free_pages(..., 1), but its error path and mock_viommu_destroy() only release the first page using free_page(), leaking the second page. Use free_pages() with the matching order instead to avoid any page leaks. Fixes: 80478a2b450e ("iommufd/selftest: Add coverage for the new mmap interface") Link: https://patch.msgid.link/r/20260312164040.457293-3-thorsten.blum@linux.dev Signed-off-by: Thorsten Blum Reviewed-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Signed-off-by: Jason Gunthorpe (cherry picked from commit 09c091fddb0b93297ea659ab48ee64f54ebeeaa2) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/selftest.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index fe39fd829aca..a6fa9eeb578c 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -605,7 +605,7 @@ static void mock_viommu_destroy(struct iommufd_viommu *viommu) if (mock_viommu->mmap_offset) iommufd_viommu_destroy_mmap(&mock_viommu->core, mock_viommu->mmap_offset); - free_page((unsigned long)mock_viommu->page); + free_pages((unsigned long)mock_viommu->page, 1); mutex_destroy(&mock_viommu->queue_mutex); /* iommufd core frees mock_viommu and viommu */ @@ -839,7 +839,7 @@ err_destroy_mmap: iommufd_viommu_destroy_mmap(&mock_viommu->core, mock_viommu->mmap_offset); err_free_page: - free_page((unsigned long)mock_viommu->page); + free_pages((unsigned long)mock_viommu->page, 1); return rc; } From 7329b0b89a64a71de29d0cd62a90962192129d88 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:27:11 -0700 Subject: [PATCH 08/67] iommu/vt-d: Fix intel iommu iotlb sync hardlockup and retry JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit fe89277c9ceb0d6af0aa665bcf24a41d8b1b79cd Author: Guanghui Feng Date: Mon Mar 16 15:16:39 2026 +0800 iommu/vt-d: Fix intel iommu iotlb sync hardlockup and retry During the qi_check_fault process after an IOMMU ITE event, requests at odd-numbered positions in the queue are set to QI_ABORT, only satisfying single-request submissions. However, qi_submit_sync now supports multiple simultaneous submissions, and can't guarantee that the wait_desc will be at an odd-numbered position. Therefore, if an item times out, IOMMU can't re-initiate the request, resulting in an infinite polling wait. This modifies the process by setting the status of all requests already fetched by IOMMU and recorded as QI_IN_USE status (including wait_desc requests) to QI_ABORT, thus enabling multiple requests to be resubmitted. Fixes: 8a1d82462540 ("iommu/vt-d: Multiple descriptors per qi_submit_sync()") Cc: stable@vger.kernel.org Signed-off-by: Guanghui Feng Tested-by: Shuai Xue Reviewed-by: Shuai Xue Reviewed-by: Samiullah Khawaja Link: https://lore.kernel.org/r/20260306101516.3885775-1-guanghuifeng@linux.alibaba.com Signed-off-by: Lu Baolu Fixes: 8a1d82462540 ("iommu/vt-d: Multiple descriptors per qi_submit_sync()") Signed-off-by: Joerg Roedel (cherry picked from commit fe89277c9ceb0d6af0aa665bcf24a41d8b1b79cd) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/dmar.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index ec975c73cfe6..6938800e9884 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1314,7 +1314,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) if (fault & DMA_FSTS_ITE) { head = readl(iommu->reg + DMAR_IQH_REG); head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH; - head |= 1; tail = readl(iommu->reg + DMAR_IQT_REG); tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; @@ -1331,7 +1330,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index) do { if (qi->desc_status[head] == QI_IN_USE) qi->desc_status[head] = QI_ABORT; - head = (head - 2 + QI_LENGTH) % QI_LENGTH; + head = (head - 1 + QI_LENGTH) % QI_LENGTH; } while (head != tail); /* From 2d76f80a7c95811a5b7123e5ed8445177d916f33 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:27:29 -0700 Subject: [PATCH 09/67] iommu/vt-d: Only handle IOPF for SVA when PRI is supported JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 39c20c4e83b9f78988541d829aa34668904e54a0 Author: Lu Baolu Date: Mon Mar 16 15:16:40 2026 +0800 iommu/vt-d: Only handle IOPF for SVA when PRI is supported In intel_svm_set_dev_pasid(), the driver unconditionally manages the IOPF handling during a domain transition. However, commit a86fb7717320 ("iommu/vt-d: Allow SVA with device-specific IOPF") introduced support for SVA on devices that handle page faults internally without utilizing the PCI PRI. On such devices, the IOMMU-side IOPF infrastructure is not required. Calling iopf_for_domain_replace() on these devices is incorrect and can lead to unexpected failures during PASID attachment or unwinding. Add a check for info->pri_supported to ensure that the IOPF queue logic is only invoked for devices that actually rely on the IOMMU's PRI-based fault handling. Fixes: 17fce9d2336d ("iommu/vt-d: Put iopf enablement in domain attach path") Cc: stable@vger.kernel.org Suggested-by: Kevin Tian Reviewed-by: Kevin Tian Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20260310075520.295104-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel (cherry picked from commit 39c20c4e83b9f78988541d829aa34668904e54a0) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/svm.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 71de7947971f..b1389df36ef8 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,9 +164,12 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (info->pri_supported) { + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; + } /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -181,7 +184,8 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - iopf_for_domain_replace(old, domain, dev); + if (info->pri_supported) + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret; From e7c63a030162a6e3ee68590cfeda6940c8b107b8 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:31:50 -0700 Subject: [PATCH 10/67] iommu/amd: Block identity domain when SNP enabled JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit ba17de98545d07285d15ce4fe2afe98283338fb0 Author: Joe Damato Date: Mon Mar 9 16:52:33 2026 -0700 iommu/amd: Block identity domain when SNP enabled Previously, commit 8388f7df936b ("iommu/amd: Do not support IOMMU_DOMAIN_IDENTITY after SNP is enabled") prevented users from changing the IOMMU domain to identity if SNP was enabled. This resulted in an error when writing to sysfs: # echo "identity" > /sys/kernel/iommu_groups/50/type -bash: echo: write error: Cannot allocate memory However, commit 4402f2627d30 ("iommu/amd: Implement global identity domain") changed the flow of the code, skipping the SNP guard and allowing users to change the IOMMU domain to identity after a machine has booted. Once the user does that, they will probably try to bind and the device/driver will start to do DMA which will trigger errors: iommu ivhd3: AMD-Vi: Event logged [ILLEGAL_DEV_TABLE_ENTRY device=0000:43:00.0 pasid=0x00000 address=0x3737b01000 flags=0x0020] iommu ivhd3: AMD-Vi: Control Reg : 0xc22000142148d AMD-Vi: DTE[0]: 6000000000000003 AMD-Vi: DTE[1]: 0000000000000001 AMD-Vi: DTE[2]: 2000003088b3e013 AMD-Vi: DTE[3]: 0000000000000000 bnxt_en 0000:43:00.0 (unnamed net_device) (uninitialized): Error (timeout: 500015) msg {0x0 0x0} len:0 iommu ivhd3: AMD-Vi: Event logged [ILLEGAL_DEV_TABLE_ENTRY device=0000:43:00.0 pasid=0x00000 address=0x3737b01000 flags=0x0020] iommu ivhd3: AMD-Vi: Control Reg : 0xc22000142148d AMD-Vi: DTE[0]: 6000000000000003 AMD-Vi: DTE[1]: 0000000000000001 AMD-Vi: DTE[2]: 2000003088b3e013 AMD-Vi: DTE[3]: 0000000000000000 bnxt_en 0000:43:00.0: probe with driver bnxt_en failed with error -16 To prevent this from happening, create an attach wrapper for identity_domain_ops which returns EINVAL if amd_iommu_snp_en is true. With this commit applied: # echo "identity" > /sys/kernel/iommu_groups/62/type -bash: echo: write error: Invalid argument Fixes: 4402f2627d30 ("iommu/amd: Implement global identity domain") Signed-off-by: Joe Damato Reviewed-by: Vasant Hegde Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel (cherry picked from commit ba17de98545d07285d15ce4fe2afe98283338fb0) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 79c043247015..101cbf9170b9 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2909,8 +2909,21 @@ static struct iommu_domain blocked_domain = { static struct protection_domain identity_domain; +static int amd_iommu_identity_attach(struct iommu_domain *dom, struct device *dev, + struct iommu_domain *old) +{ + /* + * Don't allow attaching a device to the identity domain if SNP is + * enabled. + */ + if (amd_iommu_snp_en) + return -EINVAL; + + return amd_iommu_attach_device(dom, dev, old); +} + static const struct iommu_domain_ops identity_domain_ops = { - .attach_dev = amd_iommu_attach_device, + .attach_dev = amd_iommu_identity_attach, }; void amd_iommu_init_identity_domain(void) From e3fcc1d40141e2373ad4f6d42fd040b640543370 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:28:28 -0700 Subject: [PATCH 11/67] dma-mapping: add missing `inline` for `dma_free_attrs` JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 2cdaff22ed26f1e619aa2b43f27bb84f2c6ef8f8 Author: Miguel Ojeda Date: Wed Mar 25 02:55:48 2026 +0100 dma-mapping: add missing `inline` for `dma_free_attrs` Under an UML build for an upcoming series [1], I got `-Wstatic-in-inline` for `dma_free_attrs`: BINDGEN rust/bindings/bindings_generated.rs - due to target missing In file included from rust/helpers/helpers.c:59: rust/helpers/dma.c:17:2: warning: static function 'dma_free_attrs' is used in an inline function with external linkage [-Wstatic-in-inline] 17 | dma_free_attrs(dev, size, cpu_addr, dma_handle, attrs); | ^ rust/helpers/dma.c:12:1: note: use 'static' to give inline function 'rust_helper_dma_free_attrs' internal linkage 12 | __rust_helper void rust_helper_dma_free_attrs(struct device *dev, size_t size, | ^ | static The issue is that `dma_free_attrs` was not marked `inline` when it was introduced alongside the rest of the stubs. Thus mark it. Fixes: ed6ccf10f24b ("dma-mapping: properly stub out the DMA API for !CONFIG_HAS_DMA") Closes: https://lore.kernel.org/rust-for-linux/20260322194616.89847-1-ojeda@kernel.org/ [1] Signed-off-by: Miguel Ojeda Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260325015548.70912-1-ojeda@kernel.org (cherry picked from commit 2cdaff22ed26f1e619aa2b43f27bb84f2c6ef8f8) Signed-off-by: Jerry Snitselaar --- include/linux/dma-mapping.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 29973baa0581..f46a0848cb24 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -248,8 +248,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, { return NULL; } -static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, unsigned long attrs) +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { } static inline void *dmam_alloc_attrs(struct device *dev, size_t size, From b654879f462e2ee9c0bf268c85c38c3d19d14742 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:29:21 -0700 Subject: [PATCH 12/67] iommufd: Fix return value of iommufd_fault_fops_write() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit aaca2aa92785a6ab8e3183e7184bca447a99cd76 Author: Zhenzhong Duan Date: Sun Mar 29 23:07:55 2026 -0400 iommufd: Fix return value of iommufd_fault_fops_write() copy_from_user() may return number of bytes failed to copy, we should not pass over this number to user space to cheat that write() succeed. Instead, -EFAULT should be returned. Link: https://patch.msgid.link/r/20260330030755.12856-1-zhenzhong.duan@intel.com Cc: stable@vger.kernel.org Fixes: 07838f7fd529 ("iommufd: Add iommufd fault object") Signed-off-by: Zhenzhong Duan Reviewed-by: Lu Baolu Reviewed-by: Pranjal Shrivastava Reviewed-by: Shuai Xue Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit aaca2aa92785a6ab8e3183e7184bca447a99cd76) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index e23d9ee4fe38..8e45752f500e 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -187,9 +187,10 @@ static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *b mutex_lock(&fault->mutex); while (count > done) { - rc = copy_from_user(&response, buf + done, response_size); - if (rc) + if (copy_from_user(&response, buf + done, response_size)) { + rc = -EFAULT; break; + } static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS == (int)IOMMU_PAGE_RESP_SUCCESS); From 2ea97e2318b123c6bc742bcc42f08fdafa83d618 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:29:36 -0700 Subject: [PATCH 13/67] iommu/vt-d: Block PASID attachment to nested domain with dirty tracking JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-53372 commit cc5bd898ff70710ffc41cd8e5c2741cb64750047 Author: Zhenzhong Duan Date: Thu Apr 2 14:57:24 2026 +0800 iommu/vt-d: Block PASID attachment to nested domain with dirty tracking Kernel lacks dirty tracking support on nested domain attached to PASID, fails the attachment early if nesting parent domain is dirty tracking configured, otherwise dirty pages would be lost. Cc: stable@vger.kernel.org Fixes: 67f6f56b5912 ("iommu/vt-d: Add set_dev_pasid callback for nested domain") Suggested-by: Kevin Tian Signed-off-by: Zhenzhong Duan Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Link: https://lore.kernel.org/r/20260330101108.12594-2-zhenzhong.duan@intel.com Signed-off-by: Lu Baolu Fixes: 67f6f56b5912 ("iommu/vt-d: Add set_dev_pasid callback for nested domain") Signed-off-by: Joerg Roedel (cherry picked from commit cc5bd898ff70710ffc41cd8e5c2741cb64750047) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/nested.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index e9a440e9c960..9ff6c72dd073 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -148,6 +148,7 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, { struct device_domain_info *info = dev_iommu_priv_get(dev); struct dmar_domain *dmar_domain = to_dmar_domain(domain); + struct iommu_domain *s2_domain = &dmar_domain->s2_domain->domain; struct intel_iommu *iommu = info->iommu; struct dev_pasid_info *dev_pasid; int ret; @@ -155,10 +156,13 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) return -EOPNOTSUPP; + if (s2_domain->dirty_ops) + return -EINVAL; + if (context_copied(iommu, info->bus, info->devfn)) return -EBUSY; - ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev); + ret = paging_domain_compatible(s2_domain, dev); if (ret) return ret; From ff7bb89c7087e04e825e6a154a697411a6279667 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:31:58 -0700 Subject: [PATCH 14/67] iommu/amd: Fix clone_alias() to use the original device's devid JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-53053 commit faad224fe0f0857a04ff2eb3c90f0de57f47d0f3 Author: Vasant Hegde Date: Wed Apr 1 08:00:17 2026 +0000 iommu/amd: Fix clone_alias() to use the original device's devid Currently clone_alias() assumes first argument (pdev) is always the original device pointer. This function is called by pci_for_each_dma_alias() which based on topology decides to send original or alias device details in first argument. This meant that the source devid used to look up and copy the DTE may be incorrect, leading to wrong or stale DTE entries being propagated to alias device. Fix this by passing the original pdev as the opaque data argument to both the direct clone_alias() call and pci_for_each_dma_alias(). Inside clone_alias(), retrieve the original device from data and compute devid from it. Fixes: 3332364e4ebc ("iommu/amd: Support multiple PCI DMA aliases in device table") Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel (cherry picked from commit faad224fe0f0857a04ff2eb3c90f0de57f47d0f3) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 101cbf9170b9..afd5c6006e9f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -403,11 +403,12 @@ struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) return NULL; } -static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) +static int clone_alias(struct pci_dev *pdev_origin, u16 alias, void *data) { struct dev_table_entry new; struct amd_iommu *iommu; struct iommu_dev_data *dev_data, *alias_data; + struct pci_dev *pdev = data; u16 devid = pci_dev_id(pdev); int ret = 0; @@ -454,9 +455,9 @@ static void clone_aliases(struct amd_iommu *iommu, struct device *dev) * part of the PCI DMA aliases if it's bus differs * from the original device. */ - clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); + clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev); - pci_for_each_dma_alias(pdev, clone_alias, NULL); + pci_for_each_dma_alias(pdev, clone_alias, pdev); } static void setup_aliases(struct amd_iommu *iommu, struct device *dev) From ede11490d7cedd7ced63b84c886fd78f000857f8 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:30:30 -0700 Subject: [PATCH 15/67] iommufd: Fix a race with concurrent allocation and unmap JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 8602018b1f17fbdaa5e5d79f4c8603ad20640c12 Author: Sina Hassani Date: Fri Apr 10 11:32:44 2026 -0700 iommufd: Fix a race with concurrent allocation and unmap iopt_unmap_iova_range() releases the lock on iova_rwsem inside the loop body when getting to the more expensive unmap operations. This is fine on its own, except the loop condition is based on the first area that matches the unmap address range. If a concurrent call to map picks an area that was unmapped in previous iterations, the loop mistakenly tries to unmap it. This is reproducible by having one userspace thread map buffers and pass them to another thread that unmaps them. The problem manifests as EBUSY errors with single page mappings. Fix this by advancing the start pointer after unmapping an area. This ensures each iteration only examines the IOVA range that remains mapped, which is guaranteed not to have overlaps. Cc: stable@vger.kernel.org Fixes: 51fe6141f0f6 ("iommufd: Data structure to provide IOVA to PFN mapping") Link: https://patch.msgid.link/r/CAAJpGJSR4r_ds1JOjmkqHtsBPyxu8GntoeW08Sk5RNQPmgi+tg@mail.gmail.com Signed-off-by: Sina Hassani Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 8602018b1f17fbdaa5e5d79f4c8603ad20640c12) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/io_pagetable.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 436992331111..4e04152129b1 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -814,6 +814,16 @@ again: unmapped_bytes += area_last - area_first + 1; down_write(&iopt->iova_rwsem); + + /* + * After releasing the iova_rwsem concurrent allocation could + * place new areas at IOVAs we have already unmapped. Keep + * moving the start of the search forward to ignore the area + * already unmapped. + */ + if (area_last >= last) + break; + start = area_last + 1; } out_unlock_iova: From 9309f5fb301a05b87b9b4d9fd10a391ec3b4ab8a Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:30:44 -0700 Subject: [PATCH 16/67] iommu/pages: Fix iommu_pages_flush_incoherent() for non-x86 JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit f9471dc1a7177f594acf8106cc4a6ab33bf0bcb6 Author: Mostafa Saleh Date: Fri Apr 24 11:50:51 2026 +0000 iommu/pages: Fix iommu_pages_flush_incoherent() for non-x86 The dma_sync_single_for_device() function expects a dma_addr_t, but iommu_pages_flush_incoherent() was incorrectly passing a virtual address. Since iommu_pages_start_incoherent() enforces a 1:1 mapping between DMA addresses and physical addresses (checked via WARN_ON), we can convert the virtual address to a physical address before passing it to the DMA API. This also matches the behaviour of the other non-x86 in iommu_pages_free_incoherent(), which uses virt_to_phys(virt); Fixes: 36ae67b13976 ("iommu/pages: Add support for incoherent IOMMU page table walkers") Signed-off-by: Mostafa Saleh Reviewed-by: Lu Baolu Reviewed-by: Pranjal Shrivastava Reviewed-by: Jason Gunthorpe Signed-off-by: Joerg Roedel (cherry picked from commit f9471dc1a7177f594acf8106cc4a6ab33bf0bcb6) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommu-pages.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index ae9da4f571f6..e9e605b5fa3a 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -137,7 +137,7 @@ static inline void iommu_pages_flush_incoherent(struct device *dma_dev, void *virt, size_t offset, size_t len) { - dma_sync_single_for_device(dma_dev, (uintptr_t)virt + offset, len, + dma_sync_single_for_device(dma_dev, virt_to_phys(virt) + offset, len, DMA_TO_DEVICE); } void iommu_pages_stop_incoherent_list(struct iommu_pages_list *list, From 2a34227660a084b7730fa344526218afd2e4571f Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 14 May 2026 17:30:58 -0700 Subject: [PATCH 17/67] iommu/amd: Fix precedence order in set_dte_passthrough() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 10161b4a791d5c4b7ea16512c1ddb133c3f8f953 Author: Weinan Liu Date: Thu Apr 30 23:28:51 2026 +0000 iommu/amd: Fix precedence order in set_dte_passthrough() Bitwise OR | operator has a higher precedence than the ternary ?: operatior. It will be incorrectly evaluated as: new->data[1] |= (FIELD_PREP(...) | dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0; Wrap the conditional operation in parentheses to enforce the correct evaluation order. Fixes: 93eee2a49c1b ("iommu/amd: Refactor logic to program the host page table in DTE") Signed-off-by: Weinan Liu Reviewed-by: Jason Gunthorpe Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel (cherry picked from commit 10161b4a791d5c4b7ea16512c1ddb133c3f8f953) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index afd5c6006e9f..f2f6b277a502 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2149,7 +2149,8 @@ static void set_dte_passthrough(struct iommu_dev_data *dev_data, new->data[0] |= DTE_FLAG_TV | DTE_FLAG_IR | DTE_FLAG_IW; new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, domain->id) | - (dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0; + (dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0); + } static void set_dte_entry(struct amd_iommu *iommu, From f4cd6c1a0c1116e3d35429615a02fb1fe9e1ff51 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 9 Jul 2026 10:17:53 -0700 Subject: [PATCH 18/67] dma-mapping: move dma_map_resource() sanity check into debug code JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-64149 commit af0c3f05866237f7592219bfe05387bc3bfc99b5 Author: Jianpeng Chang Date: Wed May 13 15:22:09 2026 +0800 dma-mapping: move dma_map_resource() sanity check into debug code dma_map_resource() uses pfn_valid() to ensure the range is not RAM. However, pfn_valid() only checks for availability of the memory map for a PFN but it does not ensure that the PFN is actually backed by RAM. On ARM64 with SPARSEMEM (128MB section granularity), MMIO addresses that share a section with RAM will falsely trigger the WARN_ON_ONCE and cause dma_map_resource() to return DMA_MAPPING_ERROR. This causes a WARNING on Raspberry Pi 4 during spi_bcm2835 probe because the SPI FIFO register (0xfe204004) falls in the same sparsemem section as the end of RAM (0xf8000000-0xfbffffff), both in section 31 (0xf8000000-0xffffffff). Move the sanity check from dma_map_resource() into debug_dma_map_phys() and replace the unreliable pfn_valid() with pfn_valid() && !PageReserved(), which correctly identifies actual usable RAM without false positives for MMIO regions that happen to have struct pages. Since dma_map_resource() is dma_map_phys(DMA_ATTR_MMIO), the check applies equally to both APIs. Any non-reserved page represents kernel memory to a sufficient degree that using DMA_ATTR_MMIO on it is almost certainly wrong and risks breaking coherency on non-coherent platforms. ZONE_DEVICE pages used for PCI P2P DMA (MEMORY_DEVICE_PCI_P2PDMA) have PageReserved set, so they will not trigger a false positive. The check no longer blocks the mapping and uses err_printk() to integrate with dma-debug filtering. Fixes: f7326196a781 ("dma-mapping: export new dma_*map_phys() interface") Reviewed-by: Robin Murphy Signed-off-by: Jianpeng Chang Reviewed-by: Leon Romanovsky Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260513072209.1486986-1-jianpeng.chang.cn@windriver.com (cherry picked from commit af0c3f05866237f7592219bfe05387bc3bfc99b5) Signed-off-by: Jerry Snitselaar --- kernel/dma/debug.c | 9 ++++++++- kernel/dma/mapping.c | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 21db33118591..fa4aac333917 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1250,7 +1250,14 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, entry->direction = direction; entry->map_err_type = MAP_ERR_NOT_CHECKED; - if (!(attrs & DMA_ATTR_MMIO)) { + if (attrs & DMA_ATTR_MMIO) { + unsigned long pfn = PHYS_PFN(phys); + + if (pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn))) + err_printk(dev, entry, + "dma_map_resource called for RAM address %pa\n", + &phys); + } else { check_for_stack(dev, phys); if (!PhysHighMem(phys)) diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 50c4c036185a..c30b5a622232 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -362,10 +362,6 @@ EXPORT_SYMBOL(dma_unmap_sg_attrs); dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && - WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) - return DMA_MAPPING_ERROR; - return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); } EXPORT_SYMBOL(dma_map_resource); From 25a3bb558513d9a1989222b8650e4d550f355f08 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 9 Jul 2026 10:18:23 -0700 Subject: [PATCH 19/67] dma-mapping: direct: fix missing mapping for THRU_HOST_BRIDGE segments JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 560000d619ef162568746ce287f0c725e24ea967 Author: Li RongQing Date: Wed Jun 3 09:37:23 2026 +0800 dma-mapping: direct: fix missing mapping for THRU_HOST_BRIDGE segments In dma_direct_map_sg(), the case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE incorrectly used 'break' instead of falling through to MAP_NONE. As a result, segments traversing the host bridge skipped the required dma_direct_map_phys() call entirely, leaving sg->dma_address uninitialized and leading to DMA failures. Fix this by using 'fallthrough;'. Fixes: a25e7962db0d79 ("PCI/P2PDMA: Refactor the p2pdma mapping helpers") Reviewed-by: Logan Gunthorpe Signed-off-by: Li RongQing Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603013723.2439-1-lirongqing@baidu.com (cherry picked from commit 560000d619ef162568746ce287f0c725e24ea967) Signed-off-by: Jerry Snitselaar --- kernel/dma/direct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index c9fa983990cd..0cfebeb8e5e4 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -466,7 +466,7 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, * must be mapped with CPU physical address and not PCI * bus addresses. */ - break; + fallthrough; case PCI_P2PDMA_MAP_NONE: sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg), sg->length, dir, attrs); From c75ac5de838ca044b0978c027464c874c4563a96 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 9 Jul 2026 10:19:08 -0700 Subject: [PATCH 20/67] dma-debug: fix physical address retrieval in debug_dma_sync_sg_for_device JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 9bfaa86b405381326c971984fd6da184c289713f Author: Li RongQing Date: Wed Jun 3 20:37:08 2026 +0800 dma-debug: fix physical address retrieval in debug_dma_sync_sg_for_device In debug_dma_sync_sg_for_device(), when iterating over a scatterlist, the debug entry population mistakenly uses the head of the scatterlist 'sg' to fetch the physical address via sg_phys(), instead of using the current iterator variable 's'. This causes dma-debug to track the physical address of the very first scatterlist entry for all subsequent entries in the list. Fix this by passing the correct loop iterator 's' to sg_phys() Fixes: 9d4f645a1fd49ee ("dma-debug: store a phys_addr_t in struct dma_debug_entry") Signed-off-by: Li RongQing Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603123708.1665-1-lirongqing@baidu.com (cherry picked from commit 9bfaa86b405381326c971984fd6da184c289713f) Signed-off-by: Jerry Snitselaar --- kernel/dma/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index fa4aac333917..03780f39613c 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1555,7 +1555,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, struct dma_debug_entry ref = { .type = dma_debug_sg, .dev = dev, - .paddr = sg_phys(sg), + .paddr = sg_phys(s), .dev_addr = sg_dma_address(s), .size = sg_dma_len(s), .direction = direction, From 2f814ca94bbabf0a4f71935778ad7d1260b561ce Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Thu, 9 Jul 2026 10:18:57 -0700 Subject: [PATCH 21/67] iommu/dma-iommu: Fix wrong scatterlist length assignment in P2PDMA path JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit db50fb87015b955a5a0c155293b2dd40d63a3b9e Author: Li RongQing Date: Sat May 30 07:28:52 2026 -0400 iommu/dma-iommu: Fix wrong scatterlist length assignment in P2PDMA path In iommu_dma_map_sg(), when handling PCI P2PDMA cases, the DMA length of the current scatterlist segment `s` is incorrectly assigned from the head entry `sg->length` instead of the current entry `s->length`. This typo causes all P2PDMA segments in the scatterlist to inherit the length of the first segment, leading to corrupted DMA lengths for multi- segment scatterlists. Fix this by using `s->length` instead of `sg->length`. Fixes: a25e7962db ("PCI/P2PDMA: Refactor the p2pdma mapping helpers") Signed-off-by: Li RongQing Reviewed-by: Logan Gunthorpe Signed-off-by: Joerg Roedel (cherry picked from commit db50fb87015b955a5a0c155293b2dd40d63a3b9e) Signed-off-by: Jerry Snitselaar --- drivers/iommu/dma-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index aeaf8fad985c..0b959da66fd6 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1441,7 +1441,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, */ s->dma_address = pci_p2pdma_bus_addr_map( p2pdma_state.mem, sg_phys(s)); - sg_dma_len(s) = sg->length; + sg_dma_len(s) = s->length; sg_dma_mark_bus_address(s); continue; default: From bccea0923f12c259a5367514802adb53f2e71bfb Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:38:30 -0700 Subject: [PATCH 22/67] dma: contiguous: Check return value of dma_contiguous_reserve_area() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit c33efdfcfa6f80e05ce1ee33694c1bad4994cd78 Author: Shanker Donthineni Date: Thu Jan 29 12:13:17 2026 -0600 dma: contiguous: Check return value of dma_contiguous_reserve_area() Commit 8f1fc1bf1a3d ("dma: contiguous: Reserve default CMA heap") introduced a bug where dma_heap_cma_register_heap() is called with a NULL pointer when dma_contiguous_reserve_area() fails to reserve the CMA area. When dma_contiguous_reserve_area() fails, dma_contiguous_default_area remains NULL (initialized as a global variable), but the code doesn't check the return value and proceeds to call dma_heap_cma_register_heap() with this NULL pointer. Later during boot, add_cma_heaps() iterates through the dma_areas[] array and attempts to register heaps. When it encounters the NULL pointer stored by the earlier call, it crashes in __add_cma_heap() -> dma_heap_add() when trying to dereference the NULL CMA pointer. The crash manifests as: Unable to handle kernel NULL pointer dereference at virtual address 0000000000000038 ... Call trace: dma_heap_add+0x40/0x2b0 __add_cma_heap+0x80/0xe0 add_cma_heaps+0x64/0xb0 do_one_initcall+0x60/0x318 kernel_init_freeable+0x260/0x2f0 kernel_init+0x2c/0x168 ret_from_fork+0x10/0x20 Fix this by checking the return value of dma_contiguous_reserve_area() and only calling dma_heap_cma_register_heap() when the reservation succeeds. Fixes: 8f1fc1bf1a3d ("dma: contiguous: Reserve default CMA heap") Signed-off-by: Shanker Donthineni Reviewed-by: T.J. Mercier Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260129181317.2429196-1-sdonthineni@nvidia.com (cherry picked from commit c33efdfcfa6f80e05ce1ee33694c1bad4994cd78) Signed-off-by: Jerry Snitselaar --- kernel/dma/contiguous.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 0807d34f6041..cb3e8b811003 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -252,10 +252,12 @@ void __init dma_contiguous_reserve(phys_addr_t limit) pr_debug("%s: reserving %ld MiB for global area\n", __func__, (unsigned long)selected_size / SZ_1M); - dma_contiguous_reserve_area(selected_size, selected_base, - selected_limit, - &dma_contiguous_default_area, - fixed); + ret = dma_contiguous_reserve_area(selected_size, selected_base, + selected_limit, + &dma_contiguous_default_area, + fixed); + if (ret) + return; ret = dma_heap_cma_register_heap(dma_contiguous_default_area); if (ret) From 4dce12bcb76e6b4cd8bbfbc8ae136cff6000eb87 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:43:59 -0700 Subject: [PATCH 23/67] iommufd: Use sizeof(*hdr) instead of sizeof(hdr) in veventq read JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit be93d186ae88a92e7aa77e122d4e661fa57b1e39 Author: Kai Aizen Date: Thu Apr 30 20:56:30 2026 +0300 iommufd: Use sizeof(*hdr) instead of sizeof(hdr) in veventq read The bound-check in iommufd_veventq_fops_read() for the normal vEVENT path uses sizeof(hdr) where the surrounding code uses sizeof(*hdr): if (!vevent_for_lost_events_header(cur) && sizeof(hdr) + cur->data_len > count - done) { hdr is declared as struct iommufd_vevent_header *, so sizeof(hdr) evaluates to the size of the pointer. Surrounding code uses sizeof(*hdr) consistently: if (done >= count || sizeof(*hdr) > count - done) { ... if (copy_to_user(buf + done, hdr, sizeof(*hdr))) { ... done += sizeof(*hdr); struct iommufd_vevent_header is currently 8 bytes (two __u32 fields, flags and sequence), so on 64-bit (sizeof(void *) == 8) the two expressions happen to be equal and the check works as intended. On 32-bit (sizeof(void *) == 4) the check under-counts the header by 4 bytes: a vEVENT whose data_len causes 8 + cur->data_len to exceed count - done while 4 + cur->data_len does not will pass the check, then the loop will copy_to_user 8 bytes of header followed by data_len bytes of payload, writing past the user-supplied buffer. It is also a latent bug for any future expansion of struct iommufd_vevent_header beyond sizeof(void *) on 64-bit; the check should not depend on the type happening to match the host pointer width. Use sizeof(*hdr) to match the rest of the function and the actual amount that will be copied. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/20260430175630.67078-1-kai.aizen.dev@gmail.com Cc: stable@vger.kernel.org Reported-by: Kai Aizen Signed-off-by: Kai Aizen Reviewed-by: Kevin Tian Reviewed-by: Nicolin Chen Reviewed-by: Lu Baolu Signed-off-by: Jason Gunthorpe (cherry picked from commit be93d186ae88a92e7aa77e122d4e661fa57b1e39) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 8e45752f500e..d66d5e27bc74 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -321,7 +321,7 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, /* If being a normal vEVENT, validate against the full size */ if (!vevent_for_lost_events_header(cur) && - sizeof(hdr) + cur->data_len > count - done) { + sizeof(*hdr) + cur->data_len > count - done) { iommufd_veventq_deliver_restore(veventq, cur); break; } From 76af61bc110cdba219516a42ff307d29a6b5e929 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:43:45 -0700 Subject: [PATCH 24/67] iommu/amd: Bounds-check devid in __rlookup_amd_iommu() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-53283 commit 07d0f496fe7ec5abe3bee7e38be709521567bb33 Author: Jose Fernandez (Anthropic) Date: Tue Apr 21 19:26:13 2026 +0000 iommu/amd: Bounds-check devid in __rlookup_amd_iommu() iommu_device_register() walks every device on the PCI bus via bus_for_each_dev() and calls amd_iommu_probe_device() for each. The inlined check_device() path computes the device's sbdf, calls rlookup_amd_iommu() to find the owning IOMMU, and only afterwards verifies devid <= pci_seg->last_bdf. __rlookup_amd_iommu() indexes rlookup_table[devid] with no bounds check of its own, so for a PCI device whose BDF is not described by the IVRS, the lookup reads past the end of the allocation before the caller's bounds check can run. This was harmless before commit e874c666b15b ("iommu/amd: Change rlookup, irq_lookup, and alias to use kvalloc()"): the table was a zeroed page-order allocation, so the over-read returned NULL and the caller's NULL check skipped the device. After that commit the table is a tight kvcalloc() and the over-read returns adjacent slab contents, which check_device() then dereferences as a struct amd_iommu *, causing a boot-time GPF. Seen on Google Compute Engine ct6e VMs, where the virtualized IVRS describes only the four TPU endpoints 00:04.0-07.0; the gVNIC at 00:08.0 (devid 0x40) indexes 56 bytes past the 456-byte allocation, into the adjacent kmalloc-512 slab object: pci 0000:00:04.0: Adding to iommu group 0 pci 0000:00:05.0: Adding to iommu group 1 pci 0000:00:06.0: Adding to iommu group 2 pci 0000:00:07.0: Adding to iommu group 3 Oops: general protection fault, probably for non-canonical address 0x3a64695f78746382: 0000 [#1] SMP NOPTI CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.22 #1 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 12/06/2025 RIP: 0010:amd_iommu_probe_device+0x54/0x3a0 Call Trace: __iommu_probe_device+0x107/0x520 probe_iommu_group+0x29/0x50 bus_for_each_dev+0x7e/0xe0 iommu_device_register+0xc9/0x240 iommu_go_to_state+0x9c0/0x1c60 amd_iommu_init+0x14/0x40 pci_iommu_init+0x16/0x60 do_one_initcall+0x47/0x2f0 Guard the array access in __rlookup_amd_iommu(). With the fix applied on 6.18.22, the gVNIC at 00:08.0 is skipped cleanly and the VM boots. Fixes: e874c666b15b ("iommu/amd: Change rlookup, irq_lookup, and alias to use kvalloc()") Cc: stable@vger.kernel.org Reported-by: Ziyuan Chen Tested-by: Ziyuan Chen Reviewed-by: Josef Bacik Assisted-by: Claude:unspecified Signed-off-by: Jose Fernandez (Anthropic) Signed-off-by: Joerg Roedel (cherry picked from commit 07d0f496fe7ec5abe3bee7e38be709521567bb33) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index f2f6b277a502..e77dcb7afcca 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -351,8 +351,12 @@ static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) struct amd_iommu_pci_seg *pci_seg; for_each_pci_segment(pci_seg) { - if (pci_seg->id == seg) - return pci_seg->rlookup_table[devid]; + if (pci_seg->id != seg) + continue; + /* IVRS may not describe every device on the bus */ + if (devid > pci_seg->last_bdf) + return NULL; + return pci_seg->rlookup_table[devid]; } return NULL; } From f897cb11e5f76c95fa68d5dcc1900a13c966d12d Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:47:24 -0700 Subject: [PATCH 25/67] iommu/vt-d: Disable DMAR for Intel Q35 IGFX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 2cda2e10dc8343ae01eae9e999a876b7e7d37861 Author: Naval Alcalá Date: Sat May 9 10:43:44 2026 +0800 iommu/vt-d: Disable DMAR for Intel Q35 IGFX Intel Q35 integrated graphics (8086:29b2) exhibits broken DMAR behaviour similar to other G4x/GM45 devices for which DMAR is already disabled via quirks. When DMAR is enabled, the system may hard lock up during boot or early device initialization, requiring a reset. Add the missing PCI ID to the existing quirk list to disable DMAR for this device. Fixes: 1f76249cc3be ("iommu/vt-d: Declare Broadwell igfx dmar support snafu") Cc: stable@vger.kernel.org Closes: https://bugzilla.kernel.org/show_bug.cgi?id=201185 Closes: https://bugzilla.kernel.org/show_bug.cgi?id=216064 Signed-off-by: Naval Alcalá Link: https://lore.kernel.org/r/20260410161622.13549-1-ari@naval.cat Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel (cherry picked from commit 2cda2e10dc8343ae01eae9e999a876b7e7d37861) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/iommu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index cce8bb36052f..4416f1b06613 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3931,6 +3931,9 @@ static void quirk_iommu_igfx(struct pci_dev *dev) disable_igfx_iommu = 1; } +/* Q35 integrated gfx dmar support is totally busted. */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x29b2, quirk_iommu_igfx); + /* G4x/GM45 integrated gfx dmar support is totally busted. */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx); From 882a1faa47f23723767cde69f68d1c8133920e5d Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:53:35 -0700 Subject: [PATCH 26/67] iommu/arm-smmu-qcom: Fix fastrpc compatible string in ACTLR client match table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit a6e1618a65d453d4e739e5898c662ccb1e3de6c0 Author: Bibek Kumar Patro Date: Thu Apr 23 15:32:30 2026 +0530 iommu/arm-smmu-qcom: Fix fastrpc compatible string in ACTLR client match table The qcom_smmu_actlr_client_of_match table contained "qcom,fastrpc" as the compatible string for applying ACTLR prefetch settings to FastRPC devices. However, "qcom,fastrpc" is the compatible string for the parent rpmsg channel node, which is not an IOMMU client — it carries no "iommus" property in the device tree and is never attached to an SMMU context bank. The actual IOMMU clients are the compute context bank (CB) child nodes, which use the compatible string "qcom,fastrpc-compute-cb". These nodes carry the "iommus" property and are probed by fastrpc_cb_driver via fastrpc_cb_probe(), which sets up the DMA mask and IOMMU mappings for each FastRPC session. The device tree structure is: fastrpc { compatible = "qcom,fastrpc"; /* rpmsg channel, no iommus */ ... compute-cb@3 { compatible = "qcom,fastrpc-compute-cb"; iommus = <&apps_smmu 0x1823 0x0>; /* actual IOMMU client */ }; }; Since qcom_smmu_set_actlr_dev() calls of_match_device() against the device being attached to the SMMU context bank, the "qcom,fastrpc" entry was never matching any device. As a result, the ACTLR prefetch settings (PREFETCH_DEEP | CPRE | CMTLB) were silently never applied for FastRPC compute context banks. Fix this by replacing "qcom,fastrpc" with "qcom,fastrpc-compute-cb" in the match table so that the ACTLR settings are correctly applied to the compute CB devices that are the true IOMMU clients. Assisted-by: Claude:claude-sonnet-4-6 Fixes: 3e35c3e725de ("iommu/arm-smmu: Add ACTLR data and support for qcom_smmu_500") Reviewed-by: Dmitry Baryshkov Reviewed-by: Shawn Guo Signed-off-by: Bibek Kumar Patro Signed-off-by: Will Deacon (cherry picked from commit a6e1618a65d453d4e739e5898c662ccb1e3de6c0) Signed-off-by: Jerry Snitselaar --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index edd41b5a3b6a..2d006049dd61 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -39,7 +39,7 @@ static const struct of_device_id qcom_smmu_actlr_client_of_match[] = { .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,adreno-smmu", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, - { .compatible = "qcom,fastrpc", + { .compatible = "qcom,fastrpc-compute-cb", .data = (const void *) (PREFETCH_DEEP | CPRE | CMTLB) }, { .compatible = "qcom,qcm2290-mdss", .data = (const void *) (PREFETCH_SHALLOW | CPRE | CMTLB) }, From 5aa927c06fe2fcfbb50e8e21e5d9d866a8e31025 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:55:18 -0700 Subject: [PATCH 27/67] iommu/amd: Fix premature break in init_iommu_one() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 283d245468a2b61c41aa8b582f25ed5615d1c304 Author: Vasant Hegde Date: Sun May 17 12:29:25 2026 +0000 iommu/amd: Fix premature break in init_iommu_one() In init_iommu_one(), when processing IOMMU EFR attributes, the code checks whether GASUP is enabled. If GASUP is not enabled, the code falls back to legacy guest IR mode and then breaks out of the switch statement. This break incorrectly skips the subsequent initialization steps that follow the GASUP check. These initializations are independent of GASUP support and must always be performed. Fix this by replacing the early break with a conditional else block, ensuring that the XTSUP check is only skipped when GASUP is not available. Fixes: a44092e326d4 ("iommu/amd: Use IVHD EFR for early initialization of IOMMU features") Reported-by: Sudheer Dantuluri Tested-by: Dheeraj Kumar Srivastava Signed-off-by: Vasant Hegde Signed-off-by: Joerg Roedel (cherry picked from commit 283d245468a2b61c41aa8b582f25ed5615d1c304) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/init.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 3dd0c2ac9274..6a5026bd8ee6 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1939,12 +1939,11 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, /* XT and GAM require GA mode. */ if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; - break; + } else { + if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) + amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; } - if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) - amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; - if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { pr_warn_once("Host Address Translation is not supported.\n"); amd_iommu_hatdis = true; From b8c8a1d7bae8ed94929d2fb4e6d671ea6e85a007 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:55:22 -0700 Subject: [PATCH 28/67] gpu: host1x: Fix iommu_map_sgtable() return value check JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 18f74762013a4b6aa6f905c4459e0f506f9c5c7b Author: Mikko Perttunen Date: Tue Apr 21 13:02:36 2026 +0900 gpu: host1x: Fix iommu_map_sgtable() return value check Commit "iommu: return full error code from iommu_map_sg[_atomic]()" changed iommu_map_sgtable() to return an ssize_t and negative values in error cases, rather than a size_t and a zero. pin_job() also was incorrectly assigning to 'int', which could cause overflows into negative values. Update pin_job() to correctly check for errors from iommu_map_sgtable. Fixes: ad8f36e4b6b1 ("iommu: return full error code from iommu_map_sg[_atomic]()") Signed-off-by: Mikko Perttunen Signed-off-by: Thierry Reding Link: https://patch.msgid.link/20260421-iommu_map_sgtable-return-v1-1-fb484c07d2a1@nvidia.com (cherry picked from commit 18f74762013a4b6aa6f905c4459e0f506f9c5c7b) Signed-off-by: Jerry Snitselaar --- drivers/gpu/host1x/job.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index 3ed49e1fd933..70bda32f1ff4 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -235,6 +235,8 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) } if (host->domain) { + ssize_t map_err; + for_each_sgtable_sg(map->sgt, sg, j) gather_size += sg->length; @@ -248,11 +250,11 @@ static unsigned int pin_job(struct host1x *host, struct host1x_job *job) goto put; } - err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), - map->sgt, IOMMU_READ); - if (err == 0) { + map_err = iommu_map_sgtable(host->domain, iova_dma_addr(&host->iova, alloc), + map->sgt, IOMMU_READ); + if (map_err < 0) { __free_iova(&host->iova, alloc); - err = -EINVAL; + err = map_err; goto put; } From 7122f754bf476bdfbe3bf49228bd34b458ea1c67 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:55:01 -0700 Subject: [PATCH 29/67] drm/tegra: Fix iommu_map_sgtable() return value check JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit b3f349517de8f4469c385ebb7bfdfcc148790c0f Author: Mikko Perttunen Date: Tue Apr 21 13:02:37 2026 +0900 drm/tegra: Fix iommu_map_sgtable() return value check Commit "iommu: return full error code from iommu_map_sg[_atomic]()" changed iommu_map_sgtable() to return an ssize_t and negative values in error cases, rather than a size_t and a zero. Update tegra_bo_iommu_map() to correctly check for errors from iommu_map_sgtable. Fixes: ad8f36e4b6b1 ("iommu: return full error code from iommu_map_sg[_atomic]()") Signed-off-by: Mikko Perttunen Signed-off-by: Thierry Reding Link: https://patch.msgid.link/20260421-iommu_map_sgtable-return-v1-2-fb484c07d2a1@nvidia.com (cherry picked from commit b3f349517de8f4469c385ebb7bfdfcc148790c0f) Signed-off-by: Jerry Snitselaar --- drivers/gpu/drm/tegra/gem.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c index 6b14f1e919eb..0f4065228cc6 100644 --- a/drivers/gpu/drm/tegra/gem.c +++ b/drivers/gpu/drm/tegra/gem.c @@ -235,6 +235,7 @@ static const struct host1x_bo_ops tegra_bo_ops = { static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) { int prot = IOMMU_READ | IOMMU_WRITE; + ssize_t size; int err; if (bo->mm) @@ -256,13 +257,15 @@ static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo) bo->iova = bo->mm->start; - bo->size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot); - if (!bo->size) { + size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot); + if (size < 0) { dev_err(tegra->drm->dev, "failed to map buffer\n"); - err = -ENOMEM; + err = size; goto remove; } + bo->size = size; + mutex_unlock(&tegra->mm_lock); return 0; From d3265beecd5a69d6fabc3db01dd11a0d5c93af9b Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:55:45 -0700 Subject: [PATCH 30/67] iommufd: Fix data_len byte-count vs element-count mismatch JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 85345becfead3255a5f875d4b4d82ea01d926239 Author: Nicolin Chen Date: Thu May 21 17:36:32 2026 -0700 iommufd: Fix data_len byte-count vs element-count mismatch kzalloc_flex() computes the allocation size. With event_data typed as u64, data_len is interpreted as a u64 element count. Yet, every caller and the read path treat data_len as a byte count. The current code over-allocates by sizeof(u64) and the __counted_by() annotation overstates the length by the same factor. Re-type event_data as u8. No functional change in user-visible behavior. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/f7665f839b9dce917d6bd394375a1cf56568d86b.1779408671.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 85345becfead3255a5f875d4b4d82ea01d926239) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/iommufd_private.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 6ac1965199e9..43fbc5bed8de 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -602,7 +602,7 @@ struct iommufd_vevent { struct iommufd_vevent_header header; struct list_head node; /* for iommufd_eventq::deliver */ ssize_t data_len; - u64 event_data[] __counted_by(data_len); + u8 event_data[] __counted_by(data_len); }; #define vevent_for_lost_events_header(vevent) \ From b1c1a05146a5989f375e2caaf6c662aee706f3c7 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:56:25 -0700 Subject: [PATCH 31/67] iommu/amd: Don't split flush for amd_iommu_domain_flush_all() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 69fe699afe1afcb730164b86c228483c2da05f94 Author: Weinan Liu Date: Thu May 28 22:31:47 2026 +0000 iommu/amd: Don't split flush for amd_iommu_domain_flush_all() We have observed multiple full invalidations occurring during device detach when we are done using the vfio-device. blocked_domain_attach_device() -> detach_device() -> amd_iommu_domain_flush_all() -> amd_iommu_domain_flush_pages(..., CMD_INV_IOMMU_ALL_PAGES_ADDRESS) while (size != 0) { -> __domain_flush_pages( flush_size /* power of 2 flush_size */) -> domain_flush_pages_v1() -> build_inv_iommu_pages() -> build_inv_address() } build_inv_address() will trigger a full invalidation if the chunk size > (1 << 51). Consequently, the guest will issue multiple full invalidations for a single call to amd_iommu_domain_flush_all() Without this patch, we will see 10 time instead of 1 time full invalidations for every amd_iommu_domain_flush_all(). Cc: stable@vger.kernel.org Fixes: a270be1b3fdf ("iommu/amd: Use only natural aligned flushes in a VM") Suggested-by: Josef Bacik Suggested-by: Jason Gunthorpe Signed-off-by: Weinan Liu Reviewed-by: Wei Wang Reviewed-by: Jason Gunthorpe Reviewed-by: Samiullah Khawaja Reviewed-by: Suravee Suthikulpanit Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel (cherry picked from commit 69fe699afe1afcb730164b86c228483c2da05f94) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e77dcb7afcca..9683b8535249 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1774,7 +1774,8 @@ void amd_iommu_domain_flush_pages(struct protection_domain *domain, { lockdep_assert_held(&domain->lock); - if (likely(!amd_iommu_np_cache)) { + if (likely(!amd_iommu_np_cache) || + size >= (1ULL<<52)) { __domain_flush_pages(domain, address, size); /* Wait until IOMMU TLB and all device IOTLB flushes are complete */ From b232dc68ab5bc4adfaaba4af9389a312da9c3d92 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:56:45 -0700 Subject: [PATCH 32/67] iommu/vt-d: Clear Present bit before tearing down scalable-mode context entry JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit f46452c3df7a8d8a5addc0926e76ef19ea7da0a0 Author: Michael Bommarito Date: Thu Jun 4 14:03:07 2026 +0800 iommu/vt-d: Clear Present bit before tearing down scalable-mode context entry device_pasid_table_teardown() zeroes the 128-bit scalable-mode context entry with context_clear_entry() while the Present bit is still set. This creates a window where the hardware can fetch a torn entry, with some fields already zeroed while Present is still set, leading to unpredictable behavior or spurious faults. The context-cache invalidation is issued only after the entry has been zeroed, and intel_pasid_free_table() then frees the PASID directory pages, so the IOMMU can keep walking a stale Present=1 entry that points at freed memory. While x86 provides strong write ordering, the compiler may reorder the two 64-bit writes to the entry, and the hardware fetch is not guaranteed to be atomic with respect to multiple CPU writes. Commit c1e4f1dccbe9d ("iommu/vt-d: Clear Present bit before tearing down context entry") fixed this exact pattern in domain_context_clear_one() and the copied-context path, but device_pasid_table_teardown() was not converted. Align it with the "Guidance to Software for Invalidations" in the VT-d spec, Section 6.5.3.3, using the same ownership handshake as the sibling fix: clear only the Present bit, flush it to the IOMMU, perform the context-cache invalidation, and only then zero the rest of the entry. Fixes: 81e921fd32161 ("iommu/vt-d: Fix NULL domain on device release") Signed-off-by: Michael Bommarito Assisted-by: Claude:claude-opus-4-7 Link: https://lore.kernel.org/r/20260528025557.3209367-1-michael.bommarito@gmail.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel (cherry picked from commit f46452c3df7a8d8a5addc0926e76ef19ea7da0a0) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/pasid.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index b63a71904cfb..8d1e61808a4c 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -748,10 +748,12 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) } did = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); intel_context_flush_no_pasid(info, context, did); + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); } static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, void *data) From a9a09d05bc11adead48991ce94363e472f61bcc9 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:56:58 -0700 Subject: [PATCH 33/67] iommu/vt-d: Fix RB-tree corruption in probe error path JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 43bd9e6d5513cb1edbafdeef146a1edc3aaced56 Author: Pranjal Shrivastava Date: Thu Jun 4 14:03:10 2026 +0800 iommu/vt-d: Fix RB-tree corruption in probe error path The info->node RB-tree member is zero-initialized via kzalloc. If a device does not support ATS, the device_rbtree_insert() call is skipped. If a subsequent probe step fails, the error path jumps to device_rbtree_remove(), which misinterprets the zeroed node as a tree root and corrupts the device RB-tree. Fix this by explicitly initializing the RB-node as empty using RB_CLEAR_NODE() during initialization and guarding the removal with RB_EMPTY_NODE(). Fixes: 4f1492efb495 ("iommu/vt-d: Revert ATS timing change to fix boot failure") Reported-by: sashiko-bot@kernel.org Closes: https://lore.kernel.org/all/20260525205628.CD4431F000E9@smtp.kernel.org/ Suggested-by: Baolu Lu Signed-off-by: Pranjal Shrivastava Link: https://lore.kernel.org/r/20260531170254.60493-2-praan@google.com Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel (cherry picked from commit 43bd9e6d5513cb1edbafdeef146a1edc3aaced56) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 4416f1b06613..0b2c0e5ab00b 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -157,7 +157,10 @@ static void device_rbtree_remove(struct device_domain_info *info) unsigned long flags; spin_lock_irqsave(&iommu->device_rbtree_lock, flags); - rb_erase(&info->node, &iommu->device_rbtree); + if (!RB_EMPTY_NODE(&info->node)) { + rb_erase(&info->node, &iommu->device_rbtree); + RB_CLEAR_NODE(&info->node); + } spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags); } @@ -3251,6 +3254,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) info->dev = dev; info->iommu = iommu; + RB_CLEAR_NODE(&info->node); if (dev_is_pci(dev)) { if (ecap_dev_iotlb_support(iommu->ecap) && pci_ats_supported(pdev) && From 17bf7f8fd66a174166229703ad3196596f007fa6 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:57:25 -0700 Subject: [PATCH 34/67] iommu/dma: Do not try to iommu_map a 0 length region in swiotlb JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-53164 commit 6ec91df8aff77e2e8fe3179c1f3fc15b43a40ba3 Author: Jason Gunthorpe Date: Mon Jun 8 15:10:04 2026 -0300 iommu/dma: Do not try to iommu_map a 0 length region in swiotlb iommu_dma_iova_link_swiotlb() processes a mapping that is unaligned in three parts, the head, middle and trailer. If the middle is empty because there are no aligned pages it will call down to iommu_map() with a 0 size which the iommupt implementation will fail as illegal. It then tries to do an error unwind and starts from the wrong spot corrupting the mapping so the eventual destruction triggers a WARN_ON. Check for 0 length and avoid mapping and use offset not 0 as the starting point to unlink. This is frequently triggered by using some kinds of thunderbolt NVMe drives that trigger forced SWIOTLB for unaligned memory. NVMe seems to pass in oddly aligned buffers for the passthrough commands from smartctl that hit this condition. Cc: stable@vger.kernel.org Fixes: 433a76207dcf ("dma-mapping: Implement link/unlink ranges API") Reported-by: Mark Lord Signed-off-by: Jason Gunthorpe Reviewed-by: Christoph Hellwig Reviewed-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/0-v1-8536728bc89f+469-swiotlb_warn_jgg@nvidia.com (cherry picked from commit 6ec91df8aff77e2e8fe3179c1f3fc15b43a40ba3) Signed-off-by: Jerry Snitselaar --- drivers/iommu/dma-iommu.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 0b959da66fd6..8e9cd5c6ff7d 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1894,12 +1894,18 @@ static int iommu_dma_iova_link_swiotlb(struct device *dev, return 0; } + /* + * After removing the partial head and tail, there may be no aligned + * middle left to map. The tail still gets bounced below. + */ size -= iova_end_pad; - error = __dma_iova_link(dev, addr + mapped, phys + mapped, size, dir, - attrs); - if (error) - goto out_unmap; - mapped += size; + if (size) { + error = __dma_iova_link(dev, addr + mapped, phys + mapped, + size, dir, attrs); + if (error) + goto out_unmap; + mapped += size; + } if (iova_end_pad) { error = iommu_dma_iova_bounce_and_link(dev, addr + mapped, @@ -1912,7 +1918,8 @@ static int iommu_dma_iova_link_swiotlb(struct device *dev, return 0; out_unmap: - dma_iova_unlink(dev, state, 0, mapped, dir, attrs); + if (mapped) + dma_iova_unlink(dev, state, offset, mapped, dir, attrs); return error; } From 104d79bff3816d5feee71b4f467062c99be1425b Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 09:58:18 -0700 Subject: [PATCH 35/67] iommu: Avoid copying the user array twice in the full-array copy helper JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit e28bee5b445178390d63f7a93a5a219063c6434e Author: Nicolin Chen Date: Wed Jun 3 14:26:55 2026 -0700 iommu: Avoid copying the user array twice in the full-array copy helper iommu_copy_struct_from_full_user_array() copies a whole user array into a kernel buffer. In the common case, where user entry_len equals destination entry size, it takes a fast path and copies the whole array with a single copy_from_user(). That fast path does not return, so it falls through into the item-by-item copy_struct_from_user() loop and copies every entry a second time. For an equal entry_len that loop is just a copy_from_user() of the same bytes, so the whole array is copied twice for no benefit. Return right after the bulk copy. The per-item loop then runs only on the slow path, where entry_len differs and each entry needs size adaption. Fixes: 4f2e59ccb698 ("iommu: Add iommu_copy_struct_from_full_user_array helper") Link: https://patch.msgid.link/r/6c9eca4ff584cb977661e97799ac6fe934e7f51c.1780521606.git.nicolinc@nvidia.com Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Nicolin Chen Reviewed-by: Lu Baolu Signed-off-by: Jason Gunthorpe (cherry picked from commit e28bee5b445178390d63f7a93a5a219063c6434e) Signed-off-by: Jerry Snitselaar --- include/linux/iommu.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 232d22de8278..8750569783e5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -546,6 +546,7 @@ iommu_copy_struct_from_full_user_array(void *kdst, size_t kdst_entry_size, user_array->entry_num * user_array->entry_len)) return -EFAULT; + return 0; } /* Copy item by item */ From 4939761c93298c6e5a9f1498bdccc110384af445 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:05:17 -0700 Subject: [PATCH 36/67] of: reserved_mem: Allow reserved_mem framework detect "cma=" kernel param JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Conflicts: context diff in fdt_scan_reserved_mem_reg_nodes() commit 0fd17e5983337231dc655e9ca0095d2ca3f47405 Author: Oreoluwa Babatunde Date: Mon Jan 26 18:13:27 2026 +0100 of: reserved_mem: Allow reserved_mem framework detect "cma=" kernel param When initializing the default cma region, the "cma=" kernel parameter takes priority over a DT defined linux,cma-default region. Hence, give the reserved_mem framework the ability to detect this so that the DT defined cma region can skip initialization accordingly. Signed-off-by: Oreoluwa Babatunde Tested-by: Joy Zou Acked-by: Rob Herring (Arm) Fixes: 8a6e02d0c00e ("of: reserved_mem: Restructure how the reserved memory regions are processed") Fixes: 2c223f7239f3 ("of: reserved_mem: Restructure call site for dma_contiguous_early_fixup()") Link: https://lore.kernel.org/r/20251210002027.1171519-1-oreoluwa.babatunde@oss.qualcomm.com [mszyprow: rebased onto v6.19-rc1, added fixes tags, added a stub for cma_skip_dt_default_reserved_mem() if no CONFIG_DMA_CMA is set] Signed-off-by: Marek Szyprowski (cherry picked from commit 0fd17e5983337231dc655e9ca0095d2ca3f47405) Signed-off-by: Jerry Snitselaar --- drivers/of/of_reserved_mem.c | 19 +++++++++++++++++-- include/linux/cma.h | 9 +++++++++ kernel/dma/contiguous.c | 16 ++++++++++------ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 2e9ea751ed2d..bef68a4916b5 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -158,7 +158,7 @@ static int __init __reserved_mem_reserve_reg(unsigned long node, phys_addr_t base, size; int len; const __be32 *prop; - bool nomap; + bool nomap, default_cma; prop = of_get_flat_dt_prop(node, "reg", &len); if (!prop) @@ -171,6 +171,12 @@ static int __init __reserved_mem_reserve_reg(unsigned long node, } nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; + default_cma = of_get_flat_dt_prop(node, "linux,cma-default", NULL); + + if (default_cma && cma_skip_dt_default_reserved_mem()) { + pr_err("Skipping dt linux,cma-default for \"cma=\" kernel param.\n"); + return -EINVAL; + } while (len >= t_len) { base = dt_mem_next_cell(dt_root_addr_cells, &prop); @@ -256,12 +262,15 @@ void __init fdt_scan_reserved_mem_reg_nodes(void) fdt_for_each_subnode(child, fdt, node) { const char *uname; + bool default_cma = of_get_flat_dt_prop(child, "linux,cma-default", NULL); prop = of_get_flat_dt_prop(child, "reg", &len); if (!prop) continue; if (!of_fdt_device_is_available(fdt, child)) continue; + if (default_cma && cma_skip_dt_default_reserved_mem()) + continue; uname = fdt_get_name(fdt, child, NULL); if (len && len % t_len != 0) { @@ -406,7 +415,7 @@ static int __init __reserved_mem_alloc_size(unsigned long node, const char *unam phys_addr_t base = 0, align = 0, size; int len; const __be32 *prop; - bool nomap; + bool nomap, default_cma; int ret; prop = of_get_flat_dt_prop(node, "size", &len); @@ -430,6 +439,12 @@ static int __init __reserved_mem_alloc_size(unsigned long node, const char *unam } nomap = of_get_flat_dt_prop(node, "no-map", NULL) != NULL; + default_cma = of_get_flat_dt_prop(node, "linux,cma-default", NULL); + + if (default_cma && cma_skip_dt_default_reserved_mem()) { + pr_err("Skipping dt linux,cma-default for \"cma=\" kernel param.\n"); + return -EINVAL; + } /* Need adjust the alignment to satisfy the CMA requirement */ if (IS_ENABLED(CONFIG_CMA) diff --git a/include/linux/cma.h b/include/linux/cma.h index 62d9c1cf6326..2e6931735880 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -57,6 +57,15 @@ extern bool cma_intersects(struct cma *cma, unsigned long start, unsigned long e extern void cma_reserve_pages_on_error(struct cma *cma); +#ifdef CONFIG_DMA_CMA +extern bool cma_skip_dt_default_reserved_mem(void); +#else +static inline bool cma_skip_dt_default_reserved_mem(void) +{ + return false; +} +#endif + #ifdef CONFIG_CMA struct folio *cma_alloc_folio(struct cma *cma, int order, gfp_t gfp); bool cma_free_folio(struct cma *cma, const struct folio *folio); diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index cb3e8b811003..1740d0965652 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -91,6 +91,16 @@ static int __init early_cma(char *p) } early_param("cma", early_cma); +/* + * cma_skip_dt_default_reserved_mem - This is called from the + * reserved_mem framework to detect if the default cma region is being + * set by the "cma=" kernel parameter. + */ +bool __init cma_skip_dt_default_reserved_mem(void) +{ + return size_cmdline != -1; +} + #ifdef CONFIG_DMA_NUMA_CMA static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; @@ -477,12 +487,6 @@ static int __init rmem_cma_setup(struct reserved_mem *rmem) struct cma *cma; int err; - if (size_cmdline != -1 && default_cma) { - pr_info("Reserved memory: bypass %s node, using cmdline CMA params instead\n", - rmem->name); - return -EBUSY; - } - if (!of_get_flat_dt_prop(node, "reusable", NULL) || of_get_flat_dt_prop(node, "no-map", NULL)) return -EINVAL; From e97c4b25907329b9cdf8c75cce8f004f520cbce3 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:08:15 -0700 Subject: [PATCH 37/67] iommu/amd: Fix illegal device-id access in IOMMU debugfs JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit e4172c5b53fba04fa48b13bc3afde809d0087a7f Author: Guanghui Feng Date: Thu Mar 19 15:37:53 2026 +0800 iommu/amd: Fix illegal device-id access in IOMMU debugfs In the current AMD IOMMU debugFS, when multiple processes use the IOMMU debugFS process simultaneously, illegal access issues can occur in the following execution flow: 1. CPU1: Sets a valid sbdf via devid_write, then checks the sbdf's validity in execution flows such as devid_show, iommu_devtbl_show, and iommu_irqtbl_show. 2. CPU2: Sets an invalid sbdf via devid_write, at which point the sbdf value is -1. 3. CPU1: accesses the IOMMU device table, IRQ table, based on the invalid SBDF value of -1, resulting in illegal access. This is especially problematic in monitoring scripts, where multiple scripts may access debugFS simultaneously, and some scripts may unexpectedly set invalid values, which triggers illegal access in debugfs. This patch modifies the execution flow of devid_show, iommu_devtbl_show, and iommu_irqtbl_show to ensure that these processes determine the validity and access based on the same device-id, thus guaranteeing correctness and robustness. Signed-off-by: Guanghui Feng Signed-off-by: Joerg Roedel (cherry picked from commit e4172c5b53fba04fa48b13bc3afde809d0087a7f) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/debugfs.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 20b04996441d..0b03e0622f67 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -197,10 +197,11 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf, static int devid_show(struct seq_file *m, void *unused) { u16 devid; + int sbdf_shadow = sbdf; - if (sbdf >= 0) { - devid = PCI_SBDF_TO_DEVID(sbdf); - seq_printf(m, "%04x:%02x:%02x.%x\n", PCI_SBDF_TO_SEGID(sbdf), + if (sbdf_shadow >= 0) { + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); + seq_printf(m, "%04x:%02x:%02x.%x\n", PCI_SBDF_TO_SEGID(sbdf_shadow), PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); } else seq_puts(m, "No or Invalid input provided\n"); @@ -237,13 +238,14 @@ static int iommu_devtbl_show(struct seq_file *m, void *unused) { struct amd_iommu_pci_seg *pci_seg; u16 seg, devid; + int sbdf_shadow = sbdf; - if (sbdf < 0) { + if (sbdf_shadow < 0) { seq_puts(m, "Enter a valid device ID to 'devid' file\n"); return 0; } - seg = PCI_SBDF_TO_SEGID(sbdf); - devid = PCI_SBDF_TO_DEVID(sbdf); + seg = PCI_SBDF_TO_SEGID(sbdf_shadow); + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); for_each_pci_segment(pci_seg) { if (pci_seg->id != seg) @@ -336,19 +338,20 @@ static int iommu_irqtbl_show(struct seq_file *m, void *unused) { struct amd_iommu_pci_seg *pci_seg; u16 devid, seg; + int sbdf_shadow = sbdf; if (!irq_remapping_enabled) { seq_puts(m, "Interrupt remapping is disabled\n"); return 0; } - if (sbdf < 0) { + if (sbdf_shadow < 0) { seq_puts(m, "Enter a valid device ID to 'devid' file\n"); return 0; } - seg = PCI_SBDF_TO_SEGID(sbdf); - devid = PCI_SBDF_TO_DEVID(sbdf); + seg = PCI_SBDF_TO_SEGID(sbdf_shadow); + devid = PCI_SBDF_TO_DEVID(sbdf_shadow); for_each_pci_segment(pci_seg) { if (pci_seg->id != seg) From c928a25130376b5052c841efe25a361d6e29cacf Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:08:35 -0700 Subject: [PATCH 38/67] iommu/amd: Fix illegal cap/mmio access in IOMMU debugfs JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 0e59645683b7b6fa20eceb21a6f420e4f7412943 Author: Guanghui Feng Date: Thu Mar 19 15:37:54 2026 +0800 iommu/amd: Fix illegal cap/mmio access in IOMMU debugfs In the current AMD IOMMU debugfs, when multiple processes simultaneously access the IOMMU mmio/cap registers using the IOMMU debugfs, illegal access issues can occur in the following execution flow: 1. CPU1: Sets a valid access address using iommu_mmio/capability_write, and verifies the access address's validity in iommu_mmio/capability_show 2. CPU2: Sets an invalid address using iommu_mmio/capability_write 3. CPU1: accesses the IOMMU mmio/cap registers based on the invalid address, resulting in an illegal access. This patch modifies the execution process to first verify the address's validity and then access it based on the same address, ensuring correctness and robustness. Signed-off-by: Guanghui Feng Signed-off-by: Joerg Roedel (cherry picked from commit 0e59645683b7b6fa20eceb21a6f420e4f7412943) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/debugfs.c | 42 +++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 0b03e0622f67..4e66473d7cea 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -26,22 +26,19 @@ static ssize_t iommu_mmio_write(struct file *filp, const char __user *ubuf, { struct seq_file *m = filp->private_data; struct amd_iommu *iommu = m->private; - int ret; - - iommu->dbg_mmio_offset = -1; + int ret, dbg_mmio_offset = iommu->dbg_mmio_offset = -1; if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_mmio_offset); + ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_mmio_offset); if (ret) return ret; - if (iommu->dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) { - iommu->dbg_mmio_offset = -1; - return -EINVAL; - } + if (dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) + return -EINVAL; + iommu->dbg_mmio_offset = dbg_mmio_offset; return cnt; } @@ -49,14 +46,16 @@ static int iommu_mmio_show(struct seq_file *m, void *unused) { struct amd_iommu *iommu = m->private; u64 value; + int dbg_mmio_offset = iommu->dbg_mmio_offset; - if (iommu->dbg_mmio_offset < 0) { + if (dbg_mmio_offset < 0 || dbg_mmio_offset > + iommu->mmio_phys_end - sizeof(u64)) { seq_puts(m, "Please provide mmio register's offset\n"); return 0; } - value = readq(iommu->mmio_base + iommu->dbg_mmio_offset); - seq_printf(m, "Offset:0x%x Value:0x%016llx\n", iommu->dbg_mmio_offset, value); + value = readq(iommu->mmio_base + dbg_mmio_offset); + seq_printf(m, "Offset:0x%x Value:0x%016llx\n", dbg_mmio_offset, value); return 0; } @@ -67,23 +66,20 @@ static ssize_t iommu_capability_write(struct file *filp, const char __user *ubuf { struct seq_file *m = filp->private_data; struct amd_iommu *iommu = m->private; - int ret; - - iommu->dbg_cap_offset = -1; + int ret, dbg_cap_offset = iommu->dbg_cap_offset = -1; if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &iommu->dbg_cap_offset); + ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_cap_offset); if (ret) return ret; /* Capability register at offset 0x14 is the last IOMMU capability register. */ - if (iommu->dbg_cap_offset > 0x14) { - iommu->dbg_cap_offset = -1; + if (dbg_cap_offset > 0x14) return -EINVAL; - } + iommu->dbg_cap_offset = dbg_cap_offset; return cnt; } @@ -91,21 +87,21 @@ static int iommu_capability_show(struct seq_file *m, void *unused) { struct amd_iommu *iommu = m->private; u32 value; - int err; + int err, dbg_cap_offset = iommu->dbg_cap_offset; - if (iommu->dbg_cap_offset < 0) { + if (dbg_cap_offset < 0 || dbg_cap_offset > 0x14) { seq_puts(m, "Please provide capability register's offset in the range [0x00 - 0x14]\n"); return 0; } - err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + iommu->dbg_cap_offset, &value); + err = pci_read_config_dword(iommu->dev, iommu->cap_ptr + dbg_cap_offset, &value); if (err) { seq_printf(m, "Not able to read capability register at 0x%x\n", - iommu->dbg_cap_offset); + dbg_cap_offset); return 0; } - seq_printf(m, "Offset:0x%x Value:0x%08x\n", iommu->dbg_cap_offset, value); + seq_printf(m, "Offset:0x%x Value:0x%08x\n", dbg_cap_offset, value); return 0; } From 0746f6550d5588071770bf0d904d9451ce20e540 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:08:56 -0700 Subject: [PATCH 39/67] iommu/amd: Remove latent out-of-bounds access in IOMMU debugfs JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-64186 commit 8dfd3d8d74435344ee8dc9237596959c8b2a6cbe Author: Eder Zulian Date: Fri Apr 10 14:55:50 2026 +0200 iommu/amd: Remove latent out-of-bounds access in IOMMU debugfs In iommu_mmio_write() and iommu_capability_write(), the variables dbg_mmio_offset and dbg_cap_offset are declared as int. However, they are populated using kstrtou32_from_user(). If a user provides a sufficiently large value, it can become a negative integer. Prior to this patch, the AMD IOMMU debugfs implementation was already protected by different mechanisms. 1. #define OFS_IN_SZ 8 ensures the user string <= 8 bytes, so e.g. 0xffffffff isn't a valid input. if (cnt > OFS_IN_SZ) return -EINVAL; 2. Implicit type promotion in iommu_mmio_write(), dbg_mmio_offset is int and iommu->mmio_phys_end is u64 if (dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) return -EINVAL; 3. The show handlers would currently catch the negative number and refuse to perform the read. Replace kstrtou32_from_user() with kstrtos32_from_user() to parse the input, and check for negative values to explicitly prevent out-of-bounds memory accesses directly in iommu_mmio_write() and iommu_capability_write(). Signed-off-by: Eder Zulian Fixes: 7a4ee419e8c1 ("iommu/amd: Add debugfs support to dump IOMMU MMIO registers") Cc: stable@vger.kernel.org Signed-off-by: Joerg Roedel (cherry picked from commit 8dfd3d8d74435344ee8dc9237596959c8b2a6cbe) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/debugfs.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 4e66473d7cea..4c53b6361314 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -31,11 +31,12 @@ static ssize_t iommu_mmio_write(struct file *filp, const char __user *ubuf, if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_mmio_offset); + ret = kstrtos32_from_user(ubuf, cnt, 0, &dbg_mmio_offset); if (ret) return ret; - if (dbg_mmio_offset > iommu->mmio_phys_end - sizeof(u64)) + if (dbg_mmio_offset < 0 || dbg_mmio_offset > + iommu->mmio_phys_end - sizeof(u64)) return -EINVAL; iommu->dbg_mmio_offset = dbg_mmio_offset; @@ -71,12 +72,12 @@ static ssize_t iommu_capability_write(struct file *filp, const char __user *ubuf if (cnt > OFS_IN_SZ) return -EINVAL; - ret = kstrtou32_from_user(ubuf, cnt, 0, &dbg_cap_offset); + ret = kstrtos32_from_user(ubuf, cnt, 0, &dbg_cap_offset); if (ret) return ret; /* Capability register at offset 0x14 is the last IOMMU capability register. */ - if (dbg_cap_offset > 0x14) + if (dbg_cap_offset < 0 || dbg_cap_offset > 0x14) return -EINVAL; iommu->dbg_cap_offset = dbg_cap_offset; From 793d705f67f1601be9da8110b9b49653ed3501d6 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:13:19 -0700 Subject: [PATCH 40/67] iommupt: Optimize the gather processing for DMA-FQ mode JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 199036ae01321651fe0e4488f9e19a28af4c5f1d Author: Jason Gunthorpe Date: Fri Feb 27 11:27:02 2026 -0400 iommupt: Optimize the gather processing for DMA-FQ mode In PT_FEAT_FLUSH_RANGE mode the gather was accumulated but never flushed and then the accumulated range was discarded by the dma-iommu code in DMA-FQ mode. This is basically optimal. However for PT_FEAT_FLUSH_RANGE_NO_GAPS the page table would push flushes that are redundant with the flush all generated by the DMA-FQ mode. Disable all range accumulation in the gather, and iommu_pt triggered flushing when in iommu_iotlb_gather_queued() indicates it is in DMA-FQ mode. Reported-by: Robin Murphy Closes: https://lore.kernel.org/r/794b6121-b66b-4819-b291-9761ed21cd83@arm.com Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Reviewed-by: Samiullah Khawaja Signed-off-by: Joerg Roedel (cherry picked from commit 199036ae01321651fe0e4488f9e19a28af4c5f1d) Signed-off-by: Jerry Snitselaar --- drivers/iommu/generic_pt/iommu_pt.h | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 7e7a6e7abdee..b773c01c5a2a 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -51,16 +51,27 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather, iommu_pages_stop_incoherent_list(free_list, iommu_table->iommu_device); - if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) && - iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) { - iommu_iotlb_sync(&iommu_table->domain, iotlb_gather); - /* - * Note that the sync frees the gather's free list, so we must - * not have any pages on that list that are covered by iova/len - */ + /* + * If running in DMA-FQ mode then the unmap will be followed by an IOTLB + * flush all so we need to optimize by never flushing the IOTLB here. + * + * For NO_GAPS the user gets to pick if flushing all or doing micro + * flushes is better for their work load by choosing DMA vs DMA-FQ + * operation. Drivers should also see shadow_on_flush. + */ + if (!iommu_iotlb_gather_queued(iotlb_gather)) { + if (pt_feature(common, PT_FEAT_FLUSH_RANGE_NO_GAPS) && + iommu_iotlb_gather_is_disjoint(iotlb_gather, iova, len)) { + iommu_iotlb_sync(&iommu_table->domain, iotlb_gather); + /* + * Note that the sync frees the gather's free list, so + * we must not have any pages on that list that are + * covered by iova/len + */ + } + iommu_iotlb_gather_add_range(iotlb_gather, iova, len); } - iommu_iotlb_gather_add_range(iotlb_gather, iova, len); iommu_pages_list_splice(free_list, &iotlb_gather->freelist); } From 43d0d8563016218c64155d3702fde162c18d317b Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:15:30 -0700 Subject: [PATCH 41/67] iommu/iova: Add NULL check in iova_magazine_free() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit fa8fb60d36375ca3166a60589a624f0d0bc9ddb5 Author: lynn Date: Sat Feb 14 08:09:19 2026 +0000 iommu/iova: Add NULL check in iova_magazine_free() When iova_domain_init_rcaches() fails to allocate an iova_magazine during the initialization of per-cpu rcaches, it jumps to out_err and calls free_iova_rcaches() for cleanup. In free_iova_rcaches(), the code iterates through all possible CPUs to free both cpu_rcache->loaded and cpu_rcache->prev. However, if the original allocation failed mid-way through the CPU loop, the pointers for the remaining CPUs remain NULL. Since kmem_cache_free() does not explicitly handle NULL pointers like kfree() does, passing these NULL pointers leads to a kernel paging request fault. Add a NULL check in iova_magazine_free() to safely handle partially initialized rcaches in error paths. Signed-off-by: lynn Signed-off-by: Joerg Roedel (cherry picked from commit fa8fb60d36375ca3166a60589a624f0d0bc9ddb5) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iova.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 18f839721813..e026be5e068b 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -611,7 +611,8 @@ static struct iova_magazine *iova_magazine_alloc(gfp_t flags) static void iova_magazine_free(struct iova_magazine *mag) { - kmem_cache_free(iova_magazine_cache, mag); + if (mag) + kmem_cache_free(iova_magazine_cache, mag); } static void From 3ee6e9a508669a533ece5bcb67f749b0df013bf5 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:16:10 -0700 Subject: [PATCH 42/67] iommupt: Directly call iommupt's unmap_range() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 99fb8afa16add85ed016baee9735231bca0c32b4 Author: Jason Gunthorpe Date: Fri Feb 27 15:30:10 2026 -0400 iommupt: Directly call iommupt's unmap_range() The common algorithm in iommupt does not require the iommu_pgsize() calculations, it can directly unmap any arbitrary range. Add a new function pointer to directly call an iommupt unmap_range op and make __iommu_unmap() call it directly. Gives about a 5% gain on single page unmappings. The function pointer is run through pt_iommu_ops instead of iommu_domain_ops to discourage using it outside iommupt. All drivers with their own page tables should continue to use the simplified map/unmap_pages() style interfaces. Reviewed-by: Samiullah Khawaja Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel (cherry picked from commit 99fb8afa16add85ed016baee9735231bca0c32b4) Signed-off-by: Jerry Snitselaar --- drivers/iommu/generic_pt/iommu_pt.h | 29 ++++------------------ drivers/iommu/iommu.c | 27 ++++++++++++++++----- include/linux/generic_pt/iommu.h | 37 ++++++++++++++++++++++++----- include/linux/iommu.h | 1 + 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index b773c01c5a2a..abde5dc961c4 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -1031,34 +1031,12 @@ start_oa: return ret; } -/** - * unmap_pages() - Make a range of IOVA empty/not present - * @domain: Domain to manipulate - * @iova: IO virtual address to start - * @pgsize: Length of each page - * @pgcount: Length of the range in pgsize units starting from @iova - * @iotlb_gather: Gather struct that must be flushed on return - * - * unmap_pages() will remove a translation created by map_pages(). It cannot - * subdivide a mapping created by map_pages(), so it should be called with IOVA - * ranges that match those passed to map_pages(). The IOVA range can aggregate - * contiguous map_pages() calls so long as no individual range is split. - * - * Context: The caller must hold a write range lock that includes - * the whole range. - * - * Returns: Number of bytes of VA unmapped. iova + res will be the point - * unmapping stopped. - */ -size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount, +static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + dma_addr_t len, struct iommu_iotlb_gather *iotlb_gather) { - struct pt_iommu *iommu_table = - container_of(domain, struct pt_iommu, domain); struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT( unmap.free_list) }; - pt_vaddr_t len = pgsize * pgcount; struct pt_range range; int ret; @@ -1073,7 +1051,6 @@ size_t DOMAIN_NS(unmap_pages)(struct iommu_domain *domain, unsigned long iova, return unmap.unmapped; } -EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(unmap_pages), "GENERIC_PT_IOMMU"); static void NS(get_info)(struct pt_iommu *iommu_table, struct pt_iommu_info *info) @@ -1121,6 +1098,7 @@ static void NS(deinit)(struct pt_iommu *iommu_table) } static const struct pt_iommu_ops NS(ops) = { + .unmap_range = NS(unmap_range), #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \ IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty) .set_dirty = NS(set_dirty), @@ -1183,6 +1161,7 @@ static int pt_iommu_init_domain(struct pt_iommu *iommu_table, domain->type = __IOMMU_DOMAIN_PAGING; domain->pgsize_bitmap = info.pgsize_bitmap; + domain->is_iommupt = true; if (pt_feature(common, PT_FEAT_DYNAMIC_TOP)) range = _pt_top_range(common, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index c64161995030..8d4d0d93235e 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dma-iommu.h" #include "iommu-priv.h" @@ -2671,13 +2672,12 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, } EXPORT_SYMBOL_GPL(iommu_map); -static size_t __iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size, - struct iommu_iotlb_gather *iotlb_gather) +static size_t +__iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, + size_t size, struct iommu_iotlb_gather *iotlb_gather) { const struct iommu_domain_ops *ops = domain->ops; size_t unmapped_page, unmapped = 0; - unsigned long orig_iova = iova; unsigned int min_pagesz; if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) @@ -2723,8 +2723,23 @@ static size_t __iommu_unmap(struct iommu_domain *domain, unmapped += unmapped_page; } - trace_unmap(orig_iova, size, unmapped); - iommu_debug_unmap_end(domain, orig_iova, size, unmapped); + return unmapped; +} + +static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size, + struct iommu_iotlb_gather *iotlb_gather) +{ + struct pt_iommu *pt = iommupt_from_domain(domain); + size_t unmapped; + + if (pt) + unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather); + else + unmapped = __iommu_unmap_domain_pgtbl(domain, iova, size, + iotlb_gather); + trace_unmap(iova, size, unmapped); + iommu_debug_unmap_end(domain, iova, size, unmapped); return unmapped; } diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index 9eefbb74efd0..f094f8f44e4e 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -66,6 +66,13 @@ struct pt_iommu { struct device *iommu_device; }; +static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain) +{ + if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt) + return NULL; + return container_of(domain, struct pt_iommu, domain); +} + /** * struct pt_iommu_info - Details about the IOMMU page table * @@ -80,6 +87,29 @@ struct pt_iommu_info { }; struct pt_iommu_ops { + /** + * @unmap_range: Make a range of IOVA empty/not present + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @len: Length of the range starting from @iova + * @iotlb_gather: Gather struct that must be flushed on return + * + * unmap_range() will remove a translation created by map_range(). It + * cannot subdivide a mapping created by map_range(), so it should be + * called with IOVA ranges that match those passed to map_pages. The + * IOVA range can aggregate contiguous map_range() calls so long as no + * individual range is split. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: Number of bytes of VA unmapped. iova + res will be the + * point unmapping stopped. + */ + size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + dma_addr_t len, + struct iommu_iotlb_gather *iotlb_gather); + /** * @set_dirty: Make the iova write dirty * @iommu_table: Table to manipulate @@ -198,10 +228,6 @@ struct pt_iommu_cfg { unsigned long iova, phys_addr_t paddr, \ size_t pgsize, size_t pgcount, \ int prot, gfp_t gfp, size_t *mapped); \ - size_t pt_iommu_##fmt##_unmap_pages( \ - struct iommu_domain *domain, unsigned long iova, \ - size_t pgsize, size_t pgcount, \ - struct iommu_iotlb_gather *iotlb_gather); \ int pt_iommu_##fmt##_read_and_clear_dirty( \ struct iommu_domain *domain, unsigned long iova, size_t size, \ unsigned long flags, struct iommu_dirty_bitmap *dirty); \ @@ -223,8 +249,7 @@ struct pt_iommu_cfg { */ #define IOMMU_PT_DOMAIN_OPS(fmt) \ .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \ - .map_pages = &pt_iommu_##fmt##_map_pages, \ - .unmap_pages = &pt_iommu_##fmt##_unmap_pages + .map_pages = &pt_iommu_##fmt##_map_pages #define IOMMU_PT_DIRTY_OPS(fmt) \ .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8750569783e5..18ab5eeae83e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -223,6 +223,7 @@ enum iommu_domain_cookie_type { struct iommu_domain { unsigned type; enum iommu_domain_cookie_type cookie_type; + bool is_iommupt; const struct iommu_domain_ops *ops; const struct iommu_dirty_ops *dirty_ops; const struct iommu_ops *owner; /* Whose domain_alloc we came from */ From 460f02fba2eb85d093ec07c9ef76a32aed01a8cf Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:18:49 -0700 Subject: [PATCH 43/67] iommupt: Avoid rewalking during map JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit d6c65b0fd6218bd21ed0be7a8d3218e8f6dc91de Author: Jason Gunthorpe Date: Fri Feb 27 15:30:11 2026 -0400 iommupt: Avoid rewalking during map Currently the core code provides a simplified interface to drivers where it fragments a requested multi-page map into single page size steps after doing all the calculations to figure out what page size is appropriate. Each step rewalks the page tables from the start. Since iommupt has a single implementation of the mapping algorithm it can internally compute each step as it goes while retaining its current position in the walk. Add a new function pt_pgsz_count() which computes the same page size fragement of a large mapping operations. Compute the next fragment when all the leaf entries of the current fragement have been written, then continue walking from the current point. The function pointer is run through pt_iommu_ops instead of iommu_domain_ops to discourage using it outside iommupt. All drivers with their own page tables should continue to use the simplified map_pages() style interfaces. Reviewed-by: Samiullah Khawaja Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Signed-off-by: Joerg Roedel (cherry picked from commit d6c65b0fd6218bd21ed0be7a8d3218e8f6dc91de) Signed-off-by: Jerry Snitselaar --- drivers/iommu/generic_pt/iommu_pt.h | 133 ++++++++++++-------- drivers/iommu/generic_pt/kunit_generic_pt.h | 12 ++ drivers/iommu/generic_pt/pt_iter.h | 22 ++++ drivers/iommu/iommu.c | 39 ++++-- include/linux/generic_pt/iommu.h | 34 ++++- 5 files changed, 175 insertions(+), 65 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index abde5dc961c4..19b6daf88f2a 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -477,6 +477,7 @@ struct pt_iommu_map_args { pt_oaddr_t oa; unsigned int leaf_pgsize_lg2; unsigned int leaf_level; + pt_vaddr_t num_leaves; }; /* @@ -529,11 +530,15 @@ static int clear_contig(const struct pt_state *start_pts, static int __map_range_leaf(struct pt_range *range, void *arg, unsigned int level, struct pt_table_p *table) { + struct pt_iommu *iommu_table = iommu_from_common(range->common); struct pt_state pts = pt_init(range, level, table); struct pt_iommu_map_args *map = arg; unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2; unsigned int start_index; pt_oaddr_t oa = map->oa; + unsigned int num_leaves; + unsigned int orig_end; + pt_vaddr_t last_va; unsigned int step; bool need_contig; int ret = 0; @@ -547,6 +552,15 @@ static int __map_range_leaf(struct pt_range *range, void *arg, _pt_iter_first(&pts); start_index = pts.index; + orig_end = pts.end_index; + if (pts.index + map->num_leaves < pts.end_index) { + /* Need to stop in the middle of the table to change sizes */ + pts.end_index = pts.index + map->num_leaves; + num_leaves = 0; + } else { + num_leaves = map->num_leaves - (pts.end_index - pts.index); + } + do { pts.type = pt_load_entry_raw(&pts); if (pts.type != PT_ENTRY_EMPTY || need_contig) { @@ -572,7 +586,40 @@ static int __map_range_leaf(struct pt_range *range, void *arg, flush_writes_range(&pts, start_index, pts.index); map->oa = oa; - return ret; + map->num_leaves = num_leaves; + if (ret || num_leaves) + return ret; + + /* range->va is not valid if we reached the end of the table */ + pts.index -= step; + pt_index_to_va(&pts); + pts.index += step; + last_va = range->va + log2_to_int(leaf_pgsize_lg2); + + if (last_va - 1 == range->last_va) { + PT_WARN_ON(pts.index != orig_end); + return 0; + } + + /* + * Reached a point where the page size changed, compute the new + * parameters. + */ + map->leaf_pgsize_lg2 = pt_compute_best_pgsize( + iommu_table->domain.pgsize_bitmap, last_va, range->last_va, oa); + map->leaf_level = + pt_pgsz_lg2_to_level(range->common, map->leaf_pgsize_lg2); + map->num_leaves = pt_pgsz_count(iommu_table->domain.pgsize_bitmap, + last_va, range->last_va, oa, + map->leaf_pgsize_lg2); + + /* Didn't finish this table level, caller will repeat it */ + if (pts.index != orig_end) { + if (pts.index != start_index) + pt_index_to_va(&pts); + return -EAGAIN; + } + return 0; } static int __map_range(struct pt_range *range, void *arg, unsigned int level, @@ -595,14 +642,9 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, if (pts.type != PT_ENTRY_EMPTY) return -EADDRINUSE; ret = pt_iommu_new_table(&pts, &map->attrs); - if (ret) { - /* - * Racing with another thread installing a table - */ - if (ret == -EAGAIN) - continue; + /* EAGAIN on a race will loop again */ + if (ret) return ret; - } } else { pts.table_lower = pt_table_ptr(&pts); /* @@ -626,10 +668,12 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, * The already present table can possibly be shared with another * concurrent map. */ - if (map->leaf_level == level - 1) - ret = pt_descend(&pts, arg, __map_range_leaf); - else - ret = pt_descend(&pts, arg, __map_range); + do { + if (map->leaf_level == level - 1) + ret = pt_descend(&pts, arg, __map_range_leaf); + else + ret = pt_descend(&pts, arg, __map_range); + } while (ret == -EAGAIN); if (ret) return ret; @@ -637,6 +681,14 @@ static int __map_range(struct pt_range *range, void *arg, unsigned int level, pt_index_to_va(&pts); if (pts.index >= pts.end_index) break; + + /* + * This level is currently running __map_range_leaf() which is + * not correct if the target level has been updated to this + * level. Have the caller invoke __map_range_leaf. + */ + if (map->leaf_level == level) + return -EAGAIN; } while (true); return 0; } @@ -808,12 +860,13 @@ static int check_map_range(struct pt_iommu *iommu_table, struct pt_range *range, static int do_map(struct pt_range *range, struct pt_common *common, bool single_page, struct pt_iommu_map_args *map) { + int ret; + /* * The __map_single_page() fast path does not support DMA_INCOHERENT * flushing to keep its .text small. */ if (single_page && !pt_feature(common, PT_FEAT_DMA_INCOHERENT)) { - int ret; ret = pt_walk_range(range, __map_single_page, map); if (ret != -EAGAIN) @@ -821,50 +874,25 @@ static int do_map(struct pt_range *range, struct pt_common *common, /* EAGAIN falls through to the full path */ } - if (map->leaf_level == range->top_level) - return pt_walk_range(range, __map_range_leaf, map); - return pt_walk_range(range, __map_range, map); + do { + if (map->leaf_level == range->top_level) + ret = pt_walk_range(range, __map_range_leaf, map); + else + ret = pt_walk_range(range, __map_range, map); + } while (ret == -EAGAIN); + return ret; } -/** - * map_pages() - Install translation for an IOVA range - * @domain: Domain to manipulate - * @iova: IO virtual address to start - * @paddr: Physical/Output address to start - * @pgsize: Length of each page - * @pgcount: Length of the range in pgsize units starting from @iova - * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO - * @gfp: GFP flags for any memory allocations - * @mapped: Total bytes successfully mapped - * - * The range starting at IOVA will have paddr installed into it. The caller - * must specify a valid pgsize and pgcount to segment the range into compatible - * blocks. - * - * On error the caller will probably want to invoke unmap on the range from iova - * up to the amount indicated by @mapped to return the table back to an - * unchanged state. - * - * Context: The caller must hold a write range lock that includes the whole - * range. - * - * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA that were - * mapped are added to @mapped, @mapped is not zerod first. - */ -int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int prot, gfp_t gfp, size_t *mapped) +static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + phys_addr_t paddr, dma_addr_t len, unsigned int prot, + gfp_t gfp, size_t *mapped) { - struct pt_iommu *iommu_table = - container_of(domain, struct pt_iommu, domain); pt_vaddr_t pgsize_bitmap = iommu_table->domain.pgsize_bitmap; struct pt_common *common = common_from_iommu(iommu_table); struct iommu_iotlb_gather iotlb_gather; - pt_vaddr_t len = pgsize * pgcount; struct pt_iommu_map_args map = { .iotlb_gather = &iotlb_gather, .oa = paddr, - .leaf_pgsize_lg2 = vaffs(pgsize), }; bool single_page = false; struct pt_range range; @@ -892,13 +920,13 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, return ret; /* Calculate target page size and level for the leaves */ - if (pt_has_system_page_size(common) && pgsize == PAGE_SIZE && - pgcount == 1) { + if (pt_has_system_page_size(common) && len == PAGE_SIZE) { PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE)); if (log2_mod(iova | paddr, PAGE_SHIFT)) return -ENXIO; map.leaf_pgsize_lg2 = PAGE_SHIFT; map.leaf_level = 0; + map.num_leaves = 1; single_page = true; } else { map.leaf_pgsize_lg2 = pt_compute_best_pgsize( @@ -907,6 +935,9 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, return -ENXIO; map.leaf_level = pt_pgsz_lg2_to_level(common, map.leaf_pgsize_lg2); + map.num_leaves = pt_pgsz_count(pgsize_bitmap, range.va, + range.last_va, paddr, + map.leaf_pgsize_lg2); } ret = check_map_range(iommu_table, &range, &map); @@ -929,7 +960,6 @@ int DOMAIN_NS(map_pages)(struct iommu_domain *domain, unsigned long iova, *mapped += map.oa - paddr; return ret; } -EXPORT_SYMBOL_NS_GPL(DOMAIN_NS(map_pages), "GENERIC_PT_IOMMU"); struct pt_unmap_args { struct iommu_pages_list free_list; @@ -1098,6 +1128,7 @@ static void NS(deinit)(struct pt_iommu *iommu_table) } static const struct pt_iommu_ops NS(ops) = { + .map_range = NS(map_range), .unmap_range = NS(unmap_range), #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER) && defined(pt_entry_is_write_dirty) && \ IS_ENABLED(CONFIG_IOMMUFD_TEST) && defined(pt_entry_make_write_dirty) diff --git a/drivers/iommu/generic_pt/kunit_generic_pt.h b/drivers/iommu/generic_pt/kunit_generic_pt.h index 68278bf15cfe..374e475f591e 100644 --- a/drivers/iommu/generic_pt/kunit_generic_pt.h +++ b/drivers/iommu/generic_pt/kunit_generic_pt.h @@ -312,6 +312,17 @@ static void test_best_pgsize(struct kunit *test) } } +static void test_pgsz_count(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, + pt_pgsz_count(SZ_4K, 0, SZ_1G - 1, 0, ilog2(SZ_4K)), + SZ_1G / SZ_4K); + KUNIT_EXPECT_EQ(test, + pt_pgsz_count(SZ_2M | SZ_4K, SZ_4K, SZ_1G - 1, SZ_4K, + ilog2(SZ_4K)), + (SZ_2M - SZ_4K) / SZ_4K); +} + /* * Check that pt_install_table() and pt_table_pa() match */ @@ -770,6 +781,7 @@ static struct kunit_case generic_pt_test_cases[] = { KUNIT_CASE_FMT(test_init), KUNIT_CASE_FMT(test_bitops), KUNIT_CASE_FMT(test_best_pgsize), + KUNIT_CASE_FMT(test_pgsz_count), KUNIT_CASE_FMT(test_table_ptr), KUNIT_CASE_FMT(test_max_va), KUNIT_CASE_FMT(test_table_radix), diff --git a/drivers/iommu/generic_pt/pt_iter.h b/drivers/iommu/generic_pt/pt_iter.h index c0d8617cce29..3e45dbde6b83 100644 --- a/drivers/iommu/generic_pt/pt_iter.h +++ b/drivers/iommu/generic_pt/pt_iter.h @@ -569,6 +569,28 @@ static inline unsigned int pt_compute_best_pgsize(pt_vaddr_t pgsz_bitmap, return pgsz_lg2; } +/* + * Return the number of pgsize_lg2 leaf entries that can be mapped for + * va to oa. This accounts for any requirement to reduce or increase the page + * size across the VA range. + */ +static inline pt_vaddr_t pt_pgsz_count(pt_vaddr_t pgsz_bitmap, pt_vaddr_t va, + pt_vaddr_t last_va, pt_oaddr_t oa, + unsigned int pgsize_lg2) +{ + pt_vaddr_t len = last_va - va + 1; + pt_vaddr_t next_pgsizes = log2_set_mod(pgsz_bitmap, 0, pgsize_lg2 + 1); + + if (next_pgsizes) { + unsigned int next_pgsize_lg2 = vaffs(next_pgsizes); + + if (log2_mod(va ^ oa, next_pgsize_lg2) == 0) + len = min(len, log2_set_mod_max(va, next_pgsize_lg2) - + va + 1); + } + return log2_div(len, pgsize_lg2); +} + #define _PT_MAKE_CALL_LEVEL(fn) \ static __always_inline int fn(struct pt_range *range, void *arg, \ unsigned int level, \ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 8d4d0d93235e..481161b2c703 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2574,14 +2574,14 @@ out_set_count: return pgsize; } -int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +static int __iommu_map_domain_pgtbl(struct iommu_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t size, int prot, gfp_t gfp) { const struct iommu_domain_ops *ops = domain->ops; unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; - phys_addr_t orig_paddr = paddr; int ret = 0; might_sleep_if(gfpflags_allow_blocking(gfp)); @@ -2638,12 +2638,9 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, /* unroll mapping in case something went wrong */ if (ret) { iommu_unmap(domain, orig_iova, orig_size - size); - } else { - trace_map(orig_iova, orig_paddr, orig_size); - iommu_debug_map(domain, orig_paddr, orig_size); + return ret; } - - return ret; + return 0; } int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) @@ -2655,6 +2652,32 @@ int iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) return ops->iotlb_sync_map(domain, iova, size); } +int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct pt_iommu *pt = iommupt_from_domain(domain); + int ret; + + if (pt) { + size_t mapped = 0; + + ret = pt->ops->map_range(pt, iova, paddr, size, prot, gfp, + &mapped); + if (ret) { + iommu_unmap(domain, iova, mapped); + return ret; + } + return 0; + } + ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp); + if (!ret) + return ret; + + trace_map(iova, paddr, size); + iommu_debug_map(domain, paddr, size); + return 0; +} + int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index f094f8f44e4e..43cc98c9c55f 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -87,6 +87,33 @@ struct pt_iommu_info { }; struct pt_iommu_ops { + /** + * @map_range: Install translation for an IOVA range + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @paddr: Physical/Output address to start + * @len: Length of the range starting from @iova + * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO + * @gfp: GFP flags for any memory allocations + * + * The range starting at IOVA will have paddr installed into it. The + * rage is automatically segmented into optimally sized table entries, + * and can have any valid alignment. + * + * On error the caller will probably want to invoke unmap on the range + * from iova up to the amount indicated by @mapped to return the table + * back to an unchanged state. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA + * that were mapped are added to @mapped, @mapped is not zerod first. + */ + int (*map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + phys_addr_t paddr, dma_addr_t len, unsigned int prot, + gfp_t gfp, size_t *mapped); + /** * @unmap_range: Make a range of IOVA empty/not present * @iommu_table: Table to manipulate @@ -224,10 +251,6 @@ struct pt_iommu_cfg { #define IOMMU_PROTOTYPES(fmt) \ phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \ dma_addr_t iova); \ - int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain, \ - unsigned long iova, phys_addr_t paddr, \ - size_t pgsize, size_t pgcount, \ - int prot, gfp_t gfp, size_t *mapped); \ int pt_iommu_##fmt##_read_and_clear_dirty( \ struct iommu_domain *domain, unsigned long iova, size_t size, \ unsigned long flags, struct iommu_dirty_bitmap *dirty); \ @@ -248,8 +271,7 @@ struct pt_iommu_cfg { * iommu_pt */ #define IOMMU_PT_DOMAIN_OPS(fmt) \ - .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \ - .map_pages = &pt_iommu_##fmt##_map_pages + .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys #define IOMMU_PT_DIRTY_OPS(fmt) \ .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty From 1251dda2ddbad70bcbcbe3c6a957a2711a1803ba Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:19:02 -0700 Subject: [PATCH 44/67] iommu/dma: Always allow DMA-FQ when iommupt provides the iommu_domain JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 1c18a1212c772b6a19e8583f2fca73f3a47b60fd Author: Jason Gunthorpe Date: Thu Mar 26 16:30:32 2026 -0300 iommu/dma: Always allow DMA-FQ when iommupt provides the iommu_domain iommupt always supports the semantics required for DMA-FQ, when drivers are converted to use it they automatically get support. Detect iommpt directly instead of using IOMMU_CAP_DEFERRED_FLUSH and remove IOMMU_CAP_DEFERRED_FLUSH from converted drivers. This will also enable DMA-FQ on RISC-V. Signed-off-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel (cherry picked from commit 1c18a1212c772b6a19e8583f2fca73f3a47b60fd) Signed-off-by: Jerry Snitselaar --- drivers/iommu/amd/iommu.c | 2 -- drivers/iommu/dma-iommu.c | 13 ++++++++++++- drivers/iommu/intel/iommu.c | 2 -- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 9683b8535249..65e169e1a8d7 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2998,8 +2998,6 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) return amdr_ivrs_remap_support; case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: return true; - case IOMMU_CAP_DEFERRED_FLUSH: - return true; case IOMMU_CAP_DIRTY_TRACKING: { struct amd_iommu *iommu = get_amd_iommu_from_dev(dev); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 8e9cd5c6ff7d..8afaa9837085 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -648,6 +649,15 @@ static void iommu_dma_init_options(struct iommu_dma_options *options, } } +static bool iommu_domain_supports_fq(struct device *dev, + struct iommu_domain *domain) +{ + /* iommupt always supports DMA-FQ */ + if (iommupt_from_domain(domain)) + return true; + return device_iommu_capable(dev, IOMMU_CAP_DEFERRED_FLUSH); +} + /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() @@ -706,7 +716,8 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, struct device *dev /* If the FQ fails we can simply fall back to strict mode */ if (domain->type == IOMMU_DOMAIN_DMA_FQ && - (!device_iommu_capable(dev, IOMMU_CAP_DEFERRED_FLUSH) || iommu_dma_init_fq(domain))) + (!iommu_domain_supports_fq(dev, domain) || + iommu_dma_init_fq(domain))) domain->type = IOMMU_DOMAIN_DMA; return iova_reserve_iommu_regions(dev, domain); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 0b2c0e5ab00b..3643e6f73964 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3211,8 +3211,6 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - case IOMMU_CAP_DEFERRED_FLUSH: - return true; case IOMMU_CAP_PRE_BOOT_PROTECTION: return dmar_platform_optin(); case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: From f65d6f635f9df4118a89453cbe4620c35a4007cc Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:19:05 -0700 Subject: [PATCH 45/67] iommu/vt-d: Restore IOMMU_CAP_CACHE_COHERENCY JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit ebfaf2bcc1902d293ed25f5a0580c96f73c47cbb Author: Alex Williamson Date: Wed Apr 8 12:44:42 2026 -0600 iommu/vt-d: Restore IOMMU_CAP_CACHE_COHERENCY In removing IOMMU_CAP_DEFERRED_FLUSH, the below referenced commit was over-eager in removing the return, resulting in the test for IOMMU_CAP_CACHE_COHERENCY falling through to an irrelevant option. Restore dropped return. Fixes: 1c18a1212c77 ("iommu/dma: Always allow DMA-FQ when iommupt provides the iommu_domain") Signed-off-by: Alex Williamson Reviewed-by: Lu Baolu Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Signed-off-by: Will Deacon (cherry picked from commit ebfaf2bcc1902d293ed25f5a0580c96f73c47cbb) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/iommu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 3643e6f73964..c1a7ad8f1c48 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3211,6 +3211,7 @@ static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: + return true; case IOMMU_CAP_PRE_BOOT_PROTECTION: return dmar_platform_optin(); case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: From e285ec0ccf95cc8c7fd0c875d0421826d5ab377f Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:18:24 -0700 Subject: [PATCH 46/67] iommupt: Check for missing PAGE_SIZE in the pgsize_bitmap JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-64151 commit 8ef3f77c440005c7f04229a75976bfc078364247 Author: Jason Gunthorpe Date: Tue May 12 13:46:16 2026 -0300 iommupt: Check for missing PAGE_SIZE in the pgsize_bitmap Sashiko pointed out that the driver could drop PAGE_SIZE from the pgsize_bitmap. That is technically allowed but nothing does it, and such an iommu_domain would not be used with the DMA API today. Still, it is against the design and it is trivial to fix up. Lift the PT_WARN_ON to the if branch and just skip the fast path. Fixes: dcd6a011a8d5 ("iommupt: Add map_pages op") Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Reviewed-by: Samiullah Khawaja Tested-by: Josua Mayer Signed-off-by: Joerg Roedel (cherry picked from commit 8ef3f77c440005c7f04229a75976bfc078364247) Signed-off-by: Jerry Snitselaar --- drivers/iommu/generic_pt/iommu_pt.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 19b6daf88f2a..4877b05291c9 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -920,8 +920,8 @@ static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, return ret; /* Calculate target page size and level for the leaves */ - if (pt_has_system_page_size(common) && len == PAGE_SIZE) { - PT_WARN_ON(!(pgsize_bitmap & PAGE_SIZE)); + if (pt_has_system_page_size(common) && len == PAGE_SIZE && + likely(pgsize_bitmap & PAGE_SIZE)) { if (log2_mod(iova | paddr, PAGE_SHIFT)) return -ENXIO; map.leaf_pgsize_lg2 = PAGE_SHIFT; From 15449da38c707ad1724603d545e3aa695f4ec050 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:22:17 -0700 Subject: [PATCH 47/67] iommu: Do not call drivers for empty gathers JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 90c5def10bea574b101b7a520c015ca81742183f Author: Jason Gunthorpe Date: Mon Mar 2 18:22:52 2026 -0400 iommu: Do not call drivers for empty gathers An empty gather is coded with start=U64_MAX, end=0 and several drivers go on to convert that to a size with: end - start + 1 Which gives 2 for an empty gather. This then causes Weird Stuff to happen (for example an UBSAN splat in VT-d) that is hopefully harmless, but maybe not. Prevent drivers from being called right in iommu_iotlb_sync(). Auditing shows that AMD, Intel, Mediatek and RSIC-V drivers all do things on these empty gathers. Further, there are several callers that can trigger empty gathers, especially in unusual conditions. For example iommu_map_nosync() will call a 0 size unmap on some error paths. Also in VFIO, iommupt and other places. Cc: stable@vger.kernel.org Reported-by: Janusz Krzysztofik Closes: https://lore.kernel.org/r/11145826.aFP6jjVeTY@jkrzyszt-mobl2.ger.corp.intel.com Signed-off-by: Jason Gunthorpe Reviewed-by: Lu Baolu Reviewed-by: Samiullah Khawaja Reviewed-by: Robin Murphy Reviewed-by: Vasant Hegde Signed-off-by: Joerg Roedel (cherry picked from commit 90c5def10bea574b101b7a520c015ca81742183f) Signed-off-by: Jerry Snitselaar --- include/linux/iommu.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 18ab5eeae83e..fd220901860f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -984,7 +984,8 @@ static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) static inline void iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *iotlb_gather) { - if (domain->ops->iotlb_sync) + if (domain->ops->iotlb_sync && + likely(iotlb_gather->start < iotlb_gather->end)) domain->ops->iotlb_sync(domain, iotlb_gather); iommu_iotlb_gather_init(iotlb_gather); From 684916cacb5e57134e923c0c895ce7cd7e9c3f9e Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 10:22:34 -0700 Subject: [PATCH 48/67] iommu: Ensure .iotlb_sync is called correctly JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 7e0548525abd2bff9694e016b6a469ccd2d5a053 Author: Robin Murphy Date: Wed Apr 8 15:40:57 2026 +0100 iommu: Ensure .iotlb_sync is called correctly Many drivers have no reason to use the iotlb_gather mechanism, but do still depend on .iotlb_sync being called to properly complete an unmap. Since the core code is now relying on the gather to detect when there is legitimately something to sync, it should also take care of encoding a successful unmap when the driver does not touch the gather itself. Fixes: 90c5def10bea ("iommu: Do not call drivers for empty gathers") Reported-by: Jon Hunter Closes: https://lore.kernel.org/r/8800a38b-8515-4bbe-af15-0dae81274bf7@nvidia.com Signed-off-by: Robin Murphy Tested-by: Jon Hunter Reviewed-by: Jason Gunthorpe Tested-by: Russell King (Oracle) Signed-off-by: Will Deacon (cherry picked from commit 7e0548525abd2bff9694e016b6a469ccd2d5a053) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommu.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 481161b2c703..599cf38bcef1 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2741,6 +2741,12 @@ __iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, pr_debug("unmapped: iova 0x%lx size 0x%zx\n", iova, unmapped_page); + /* + * If the driver itself isn't using the gather, make sure + * it looks non-empty so iotlb_sync will still be called. + */ + if (iotlb_gather->start >= iotlb_gather->end) + iommu_iotlb_gather_add_range(iotlb_gather, iova, size); iova += unmapped_page; unmapped += unmapped_page; From ab6e8d5afdb4bfef0525b7d46c421b08cccbbbd1 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:16:38 -0700 Subject: [PATCH 49/67] iommu: Fix loss of errno on map failure for classic ops JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 6fc7e8a3b8115294f60f5c89de27330bf1b9c98e Author: Jason Gunthorpe Date: Tue May 12 13:46:13 2026 -0300 iommu: Fix loss of errno on map failure for classic ops A typo, likely from a rebase, inverted the condition and caused errors to be lost. Fix it to be "if (ret)". This was breaking iommu_create_device_direct_mappings() on drivers that don't use iommupt and don't fully set up their domain in alloc_pages() (i.e., SMMUv2). In this case the first call of iommu_create_device_direct_mappings() should fail due to the incompletely initialized domain. Since it wrongly returns success, the second call to iommu_create_device_direct_mappings() doesn't happen and IOMMU_RESV_DIRECT is never set up. Cc: stable@vger.kernel.org Fixes: d6c65b0fd621 ("iommupt: Avoid rewalking during map") Reported-by: Josua Mayer Closes: https://lore.kernel.org/all/321c2e57-6a17-4aef-ba42-d2ebd577e472@solid-run.com/ Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Reviewed-by: Samiullah Khawaja Reviewed-by: Mostafa Saleh Tested-by: Josua Mayer Signed-off-by: Joerg Roedel (cherry picked from commit 6fc7e8a3b8115294f60f5c89de27330bf1b9c98e) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 599cf38bcef1..abed25d2c7e8 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2670,7 +2670,7 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, return 0; } ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp); - if (!ret) + if (ret) return ret; trace_map(iova, paddr, size); From 37a534197dec1fc4d305d45bb1e19d354d38a4c7 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:16:54 -0700 Subject: [PATCH 50/67] iommu: Fix up map/unmap debugging for iommupt domains JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit b948a87228482235afbaf5f4d8037860b5c470fd Author: Jason Gunthorpe Date: Tue May 12 13:46:14 2026 -0300 iommu: Fix up map/unmap debugging for iommupt domains Sashiko noticed a few issues in this path, and a few more were found on review. Tidy them up further. These are intertwined because the debug code depends on some of the WARN_ONs to function right: Lift into iommu_map_nosync(): - The might_sleep_if() - 0 pgsize_bitmap WARN_ON - Promote the illegal domain->type to a WARN_ON - WARN_ON for illegal gfp flags Then remove the return 0 since it is now safe to call iommu_debug_map(). Lift into __iommu_unmap(): - 0 pgsize_bitmap WARN_ON - Promote the illegal domain->type to a WARN_ON - iommu_debug_unmap_begin() This now pairs with the unconditional iommu_debug_map() on the mapping side. Thus iommu debugging now works for iommupt along with some of the other debugging features. Fixes: 99fb8afa16ad ("iommupt: Directly call iommupt's unmap_range()") Fixes: d6c65b0fd621 ("iommupt: Avoid rewalking during map") Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Reviewed-by: Samiullah Khawaja Reviewed-by: Mostafa Saleh Tested-by: Josua Mayer Signed-off-by: Joerg Roedel (cherry picked from commit b948a87228482235afbaf5f4d8037860b5c470fd) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommu.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index abed25d2c7e8..c35b9feb6209 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2584,19 +2584,9 @@ static int __iommu_map_domain_pgtbl(struct iommu_domain *domain, size_t orig_size = size; int ret = 0; - might_sleep_if(gfpflags_allow_blocking(gfp)); - - if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) - return -EINVAL; - - if (WARN_ON(!ops->map_pages || domain->pgsize_bitmap == 0UL)) + if (WARN_ON(!ops->map_pages)) return -ENODEV; - /* Discourage passing strange GFP flags */ - if (WARN_ON_ONCE(gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | - __GFP_HIGHMEM))) - return -EINVAL; - /* find out the minimum page size supported */ min_pagesz = 1 << __ffs(domain->pgsize_bitmap); @@ -2658,6 +2648,15 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, struct pt_iommu *pt = iommupt_from_domain(domain); int ret; + might_sleep_if(gfpflags_allow_blocking(gfp)); + + /* Discourage passing strange GFP flags or illegal domains */ + if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING) || + !domain->pgsize_bitmap || + (gfp & (__GFP_COMP | __GFP_DMA | __GFP_DMA32 | + __GFP_HIGHMEM)))) + return -EINVAL; + if (pt) { size_t mapped = 0; @@ -2667,11 +2666,12 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, iommu_unmap(domain, iova, mapped); return ret; } - return 0; + } else { + ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, + gfp); + if (ret) + return ret; } - ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, gfp); - if (ret) - return ret; trace_map(iova, paddr, size); iommu_debug_map(domain, paddr, size); @@ -2703,10 +2703,7 @@ __iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, size_t unmapped_page, unmapped = 0; unsigned int min_pagesz; - if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) - return 0; - - if (WARN_ON(!ops->unmap_pages || domain->pgsize_bitmap == 0UL)) + if (WARN_ON(!ops->unmap_pages)) return 0; /* find out the minimum page size supported */ @@ -2725,8 +2722,6 @@ __iommu_unmap_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); - iommu_debug_unmap_begin(domain, iova, size); - /* * Keep iterating until we either unmap 'size' bytes (or more) * or we hit an area that isn't mapped. @@ -2762,6 +2757,12 @@ static size_t __iommu_unmap(struct iommu_domain *domain, unsigned long iova, struct pt_iommu *pt = iommupt_from_domain(domain); size_t unmapped; + if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING) || + !domain->pgsize_bitmap)) + return 0; + + iommu_debug_unmap_begin(domain, iova, size); + if (pt) unmapped = pt->ops->unmap_range(pt, iova, size, iotlb_gather); else From ee6cb7d57d9b2585673d5f2701011812201893ac Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:17:07 -0700 Subject: [PATCH 51/67] iommu: Handle unmap error when iommu_debug is enabled JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git CVE: CVE-2026-64152 commit 0735c54804c709d1b292f3b6947cfb560b2ce552 Author: Jason Gunthorpe Date: Tue May 12 13:46:15 2026 -0300 iommu: Handle unmap error when iommu_debug is enabled Sashiko noticed a latent bug where the map error flow called iommu_unmap() which calls iommu_debug_unmap_begin()/iommu_debug_unmap_end() however since this is an error path the map flow never actually established the original iommu_debug_map() it will malfunction. Lift the unmap error handling into iommu_map_nosync() and reorder it so the trace_map()/iommu_debug_map() records the partial mapping and then immediately unmaps it. This avoid creating the unbalanced tracking and provides saner tracing instead of a unmap unmatched to any map. Fixes: ccc21213f013 ("iommu: Add calls for IOMMU_DEBUG_PAGEALLOC") Signed-off-by: Jason Gunthorpe Reviewed-by: Pranjal Shrivastava Reviewed-by: Samiullah Khawaja Reviewed-by: Mostafa Saleh Tested-by: Josua Mayer Signed-off-by: Joerg Roedel (cherry picked from commit 0735c54804c709d1b292f3b6947cfb560b2ce552) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommu.c | 49 +++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index c35b9feb6209..2a4445c147f3 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2576,12 +2576,11 @@ out_set_count: static int __iommu_map_domain_pgtbl(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, - size_t size, int prot, gfp_t gfp) + size_t size, int prot, gfp_t gfp, + size_t *mapped) { const struct iommu_domain_ops *ops = domain->ops; - unsigned long orig_iova = iova; unsigned int min_pagesz; - size_t orig_size = size; int ret = 0; if (WARN_ON(!ops->map_pages)) @@ -2604,31 +2603,25 @@ static int __iommu_map_domain_pgtbl(struct iommu_domain *domain, pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); while (size) { - size_t pgsize, count, mapped = 0; + size_t pgsize, count, op_mapped = 0; pgsize = iommu_pgsize(domain, iova, paddr, size, &count); pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n", iova, &paddr, pgsize, count); ret = ops->map_pages(domain, iova, paddr, pgsize, count, prot, - gfp, &mapped); + gfp, &op_mapped); /* * Some pages may have been mapped, even if an error occurred, * so we should account for those so they can be unmapped. */ - size -= mapped; - + *mapped += op_mapped; if (ret) - break; + return ret; - iova += mapped; - paddr += mapped; - } - - /* unroll mapping in case something went wrong */ - if (ret) { - iommu_unmap(domain, orig_iova, orig_size - size); - return ret; + size -= op_mapped; + iova += op_mapped; + paddr += op_mapped; } return 0; } @@ -2646,6 +2639,7 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct pt_iommu *pt = iommupt_from_domain(domain); + size_t mapped = 0; int ret; might_sleep_if(gfpflags_allow_blocking(gfp)); @@ -2657,24 +2651,19 @@ int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, __GFP_HIGHMEM)))) return -EINVAL; - if (pt) { - size_t mapped = 0; - + if (pt) ret = pt->ops->map_range(pt, iova, paddr, size, prot, gfp, &mapped); - if (ret) { - iommu_unmap(domain, iova, mapped); - return ret; - } - } else { + else ret = __iommu_map_domain_pgtbl(domain, iova, paddr, size, prot, - gfp); - if (ret) - return ret; - } + gfp, &mapped); - trace_map(iova, paddr, size); - iommu_debug_map(domain, paddr, size); + trace_map(iova, paddr, mapped); + iommu_debug_map(domain, paddr, mapped); + if (ret) { + iommu_unmap(domain, iova, mapped); + return ret; + } return 0; } From 2b2299c3843ca269ed988594730bf5ae9c1c7289 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:17:24 -0700 Subject: [PATCH 52/67] iommupt: Fix the end_index calculation in __map_range_leaf() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 58829512ad461af8f35941069c209941e3a97b65 Author: Jason Gunthorpe Date: Tue May 12 13:46:17 2026 -0300 iommupt: Fix the end_index calculation in __map_range_leaf() Sashiko noticed a mismatch of units in this math: num_leaves is actually the number of leaf *entries* (so a 16-item contiguous leaf is one num_leaves), while index is in items. The mismatch in maths causes __map_range_leaf() to exit early instead of efficiently filling a larger range of contiguous PTEs. The early exit is caught by the functions above and then __map_range_leaf() is re-invoked, so there is no functional issue. Correct the misuse of units by adjusting num_leaves with the leaf size and avoid the performance cost of looping externally. There are also some mismatched types for num_leaves; simplify things to remove the duplicated calculations. Fixes: d6c65b0fd621 ("iommupt: Avoid rewalking during map") Signed-off-by: Jason Gunthorpe Reviewed-by: Samiullah Khawaja Reviewd-by: Pranjal Shrivastava Tested-by: Josua Mayer Signed-off-by: Joerg Roedel (cherry picked from commit 58829512ad461af8f35941069c209941e3a97b65) Signed-off-by: Jerry Snitselaar --- drivers/iommu/generic_pt/iommu_pt.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h index 4877b05291c9..dc91fb4e2f61 100644 --- a/drivers/iommu/generic_pt/iommu_pt.h +++ b/drivers/iommu/generic_pt/iommu_pt.h @@ -534,10 +534,12 @@ static int __map_range_leaf(struct pt_range *range, void *arg, struct pt_state pts = pt_init(range, level, table); struct pt_iommu_map_args *map = arg; unsigned int leaf_pgsize_lg2 = map->leaf_pgsize_lg2; + unsigned int leaves_avail; unsigned int start_index; pt_oaddr_t oa = map->oa; - unsigned int num_leaves; + pt_vaddr_t num_leaves; unsigned int orig_end; + unsigned int step_lg2; pt_vaddr_t last_va; unsigned int step; bool need_contig; @@ -546,21 +548,25 @@ static int __map_range_leaf(struct pt_range *range, void *arg, PT_WARN_ON(map->leaf_level != level); PT_WARN_ON(!pt_can_have_leaf(&pts)); - step = log2_to_int_t(unsigned int, - leaf_pgsize_lg2 - pt_table_item_lg2sz(&pts)); - need_contig = leaf_pgsize_lg2 != pt_table_item_lg2sz(&pts); + step_lg2 = leaf_pgsize_lg2 - pt_table_item_lg2sz(&pts); + step = log2_to_int_t(unsigned int, step_lg2); + need_contig = step_lg2 != 0; _pt_iter_first(&pts); start_index = pts.index; orig_end = pts.end_index; - if (pts.index + map->num_leaves < pts.end_index) { + leaves_avail = + log2_div_t(unsigned int, pts.end_index - pts.index, step_lg2); + if (map->num_leaves <= leaves_avail) { /* Need to stop in the middle of the table to change sizes */ - pts.end_index = pts.index + map->num_leaves; + pts.end_index = pts.index + log2_mul(map->num_leaves, step_lg2); num_leaves = 0; } else { - num_leaves = map->num_leaves - (pts.end_index - pts.index); + num_leaves = map->num_leaves - leaves_avail; } + PT_WARN_ON( + log2_mod_t(unsigned int, pts.end_index - pts.index, step_lg2)); do { pts.type = pt_load_entry_raw(&pts); if (pts.type != PT_ENTRY_EMPTY || need_contig) { From f7653b45681b44805f86e9534efcf9bc543892b4 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:28:13 -0700 Subject: [PATCH 53/67] iommufd: Move vevent memory allocation outside spinlock JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Conflicts: 69050f8d6d07 and bf4afc53b77a not backported to iommu subsystem yet commit 47443565d10c51366c9382dbc8597cd6c460b8a2 Author: Nicolin Chen Date: Thu May 21 17:36:33 2026 -0700 iommufd: Move vevent memory allocation outside spinlock The veventq memory allocation happens inside the spinlock. Given its depth is decided by the user space, this leaves a vulnerability, where userspace can allocate large queues to exhaust atomic memory reserves. Move the allocation outside the spinlock and use GFP_NOWAIT, which can fail fast under memory pressure without dipping into the GFP_ATOMIC reserves or direct-reclaiming from the threaded IRQ handler. On allocation failure, queue the lost_events_header (so userspace learns of the drop) and return -ENOMEM so the caller learns of the kernel-side memory pressure. This is intentionally distinct from the queue-overflow path, which also queues the lost_events_header but returns 0: a full queue is an expected userspace-pacing condition rather than a kernel error. A subsequent change will cap the upper bound of the veventq_depth. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/5ff36b5d80f7f6299f851be532a5195c1d2f1dae.1779408671.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 47443565d10c51366c9382dbc8597cd6c460b8a2) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/driver.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/iommufd/driver.c b/drivers/iommu/iommufd/driver.c index 21d4a35538f6..18a5899d17c0 100644 --- a/drivers/iommu/iommufd/driver.c +++ b/drivers/iommu/iommufd/driver.c @@ -149,15 +149,18 @@ int iommufd_viommu_report_event(struct iommufd_viommu *viommu, goto out_unlock_veventqs; } - spin_lock(&veventq->common.lock); - if (veventq->num_events == veventq->depth) { + /* Pre-allocate to avoid GFP_ATOMIC; use GFP_NOWAIT to avoid sleeping */ + vevent = kzalloc(struct_size(vevent, event_data, data_len), GFP_NOWAIT); + if (!vevent) { + spin_lock(&veventq->common.lock); vevent = &veventq->lost_events_header; + rc = -ENOMEM; goto out_set_header; } - vevent = kzalloc(struct_size(vevent, event_data, data_len), GFP_ATOMIC); - if (!vevent) { - rc = -ENOMEM; + spin_lock(&veventq->common.lock); + if (veventq->num_events == veventq->depth) { + kfree(vevent); vevent = &veventq->lost_events_header; goto out_set_header; } From 4d95372297d5250598c2acf585608b2c4128e937 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:28:01 -0700 Subject: [PATCH 54/67] iommufd: Set veventq_depth upper bound JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 6ebf2eb46fbd5b40393ff8fbb847ba96925beaff Author: Nicolin Chen Date: Thu May 21 17:36:34 2026 -0700 iommufd: Set veventq_depth upper bound iommufd_veventq_alloc() accepts any !0 veventq_depth from userspace, with an upper bound at U32_MAX. This leaves a vulnerability where userspace can allocate excessively large queues to exhaust kernel memory reserves. Cap the veventq_depth (maximum number of entries) to 1 << 19, matching the maximum number of entries in the SMMUv3 EVTQ (the largest use case today). Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/8426cbaa5e8294472ec7f076ef427cc473be5985.1779408671.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 6ebf2eb46fbd5b40393ff8fbb847ba96925beaff) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index d66d5e27bc74..3cd7d6d15bbd 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -473,6 +473,9 @@ int iommufd_fault_iopf_handler(struct iopf_group *group) static const struct file_operations iommufd_veventq_fops = INIT_EVENTQ_FOPS(iommufd_veventq_fops_read, NULL); +/* An arbitrary upper bound for veventq_depth that fits all existing HWs */ +#define VEVENTQ_MAX_DEPTH (1U << 19) + int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd) { struct iommu_veventq_alloc *cmd = ucmd->cmd; @@ -484,7 +487,7 @@ int iommufd_veventq_alloc(struct iommufd_ucmd *ucmd) if (cmd->flags || cmd->__reserved || cmd->type == IOMMU_VEVENTQ_TYPE_DEFAULT) return -EOPNOTSUPP; - if (!cmd->veventq_depth) + if (!cmd->veventq_depth || cmd->veventq_depth > VEVENTQ_MAX_DEPTH) return -EINVAL; viommu = iommufd_get_viommu(ucmd, cmd->viommu_id); From 38165d6cca05f2b5e3faa87e181bba10d5a4887a Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:28:32 -0700 Subject: [PATCH 55/67] iommufd: Rewind header length in done if iommufd_veventq_fops_read() fails JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 01e41ad76c12ae5c49ab4ef4fc7dd54e9b8784d6 Author: Nicolin Chen Date: Mon Jun 1 13:42:32 2026 -0700 iommufd: Rewind header length in done if iommufd_veventq_fops_read() fails When the first event copy fails, rc = -EFAULT will not be reported as done is set to the length of the copied header. Rewind it to report rc correctly. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/78f8caeb6a5d667a26b870e3068cec47dd4b5be1.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 01e41ad76c12ae5c49ab4ef4fc7dd54e9b8784d6) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 3cd7d6d15bbd..0584fdac6b7a 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -336,6 +336,7 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, if (cur->data_len && copy_to_user(buf + done, cur->event_data, cur->data_len)) { iommufd_veventq_deliver_restore(veventq, cur); + done -= sizeof(*hdr); rc = -EFAULT; break; } From de55d2586d9c388f62928a1effd43529a2c59e97 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:31:56 -0700 Subject: [PATCH 56/67] iommufd: Reject invalid read count in iommufd_veventq_fops_read() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 00203ca8323f9714630408c19a209b52397975e6 Author: Nicolin Chen Date: Mon Jun 1 13:42:33 2026 -0700 iommufd: Reject invalid read count in iommufd_veventq_fops_read() The read count must be large enough to hold a vEVENT header. For a normal vEVENT, it must also hold the trailing data following the header. iommufd_veventq_fops_read() does not validate the count, but returns 0 as if the read had succeeded while leaving the pending event in the queue. Return -EINVAL in both undersize cases. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/e1111adcc8a8882fbfd84accd6674dc846dc5689.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 00203ca8323f9714630408c19a209b52397975e6) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 0584fdac6b7a..b6123de99dc0 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -310,6 +310,9 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, if (*ppos) return -ESPIPE; + /* Minimum read count is a vEVENT header */ + if (count < sizeof(*hdr)) + return -EINVAL; while ((cur = iommufd_veventq_deliver_fetch(veventq))) { /* Validate the remaining bytes against the header size */ @@ -323,6 +326,9 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, if (!vevent_for_lost_events_header(cur) && sizeof(*hdr) + cur->data_len > count - done) { iommufd_veventq_deliver_restore(veventq, cur); + /* Read count doesn't fit a single normal vEVENT */ + if (done == 0) + rc = -EINVAL; break; } From 441610b7df1bdaad57a02450aa9ded78f9e761b4 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:31:45 -0700 Subject: [PATCH 57/67] iommufd: Propagate allocation failure in iommufd_veventq_deliver_fetch() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git Conflicts: 69050f8d6d07 and bf4afc53b77a not backported to iommu subsystem yet commit 489e63dd120bad52eba63f5506c214750cd5bc75 Author: Nicolin Chen Date: Mon Jun 1 13:42:34 2026 -0700 iommufd: Propagate allocation failure in iommufd_veventq_deliver_fetch() When the kzalloc_obj() fails in iommufd_veventq_deliver_fetch(), it returns NULL, falsely advertising to userspace that the queue is empty. Propagate the -ENOMEM properly to the caller. Fixes: e36ba5ab808e ("iommufd: Add IOMMUFD_OBJ_VEVENTQ and IOMMUFD_CMD_VEVENTQ_ALLOC") Link: https://patch.msgid.link/r/25d29feac909e36f78c145fa99ef2d4cb7a415da.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 489e63dd120bad52eba63f5506c214750cd5bc75) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index b6123de99dc0..192b6d917b5a 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -264,8 +264,10 @@ iommufd_veventq_deliver_fetch(struct iommufd_veventq *veventq) /* Make a copy of the lost_events_header for copy_to_user */ if (next == &veventq->lost_events_header) { vevent = kzalloc(sizeof(*vevent), GFP_ATOMIC); - if (!vevent) + if (!vevent) { + vevent = ERR_PTR(-ENOMEM); goto out_unlock; + } } list_del(&next->node); if (vevent) @@ -315,6 +317,12 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf, return -EINVAL; while ((cur = iommufd_veventq_deliver_fetch(veventq))) { + if (IS_ERR(cur)) { + if (done == 0) + rc = PTR_ERR(cur); + break; + } + /* Validate the remaining bytes against the header size */ if (done >= count || sizeof(*hdr) > count - done) { iommufd_veventq_deliver_restore(veventq, cur); From 5445d2a93d31e172983db0d718603eac82c6c7d3 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:32:20 -0700 Subject: [PATCH 58/67] iommufd: Reject invalid read count in iommufd_fault_fops_read() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 47916a54eeb2a9e654512ee609f71bd5b29db702 Author: Nicolin Chen Date: Mon Jun 1 13:42:35 2026 -0700 iommufd: Reject invalid read count in iommufd_fault_fops_read() The read count must be large enough to hold one fault or a group's faults. iommufd_fault_fops_read() does not validate the count, but returns 0 as if the read had succeeded while leaving the pending fault in the queue. Return -EINVAL in the undersize cases. Fixes: 07838f7fd529 ("iommufd: Add iommufd fault object") Link: https://patch.msgid.link/r/85c118a606fbedc5c132a1f5ec223a5ba23b92d2.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 47916a54eeb2a9e654512ee609f71bd5b29db702) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 192b6d917b5a..9388e8a2e42e 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -142,6 +142,9 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, if (done >= count || group->fault_count * fault_size > count - done) { iommufd_fault_deliver_restore(fault, group); + /* Read count doesn't fit the first fault group */ + if (done == 0) + rc = -EINVAL; break; } From a9beade78c458131686bfc5b9091f7a29f9ccf67 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:32:37 -0700 Subject: [PATCH 59/67] iommufd: Break the loop on failure in iommufd_fault_fops_read() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 172fc8b19825a0f5884c38f2289188284e2d45ee Author: Nicolin Chen Date: Mon Jun 1 13:42:36 2026 -0700 iommufd: Break the loop on failure in iommufd_fault_fops_read() On a copy_to_user() failure inside the inner list_for_each_entry, only the inner loop breaks; the outer while re-fetches the just-restored fault group and retries the failing copy_to_user() forever, spinning the reader at 100% CPU with fault->mutex held. Check rc after the inner loop and break the outer while as well. Fixes: 07838f7fd529 ("iommufd: Add iommufd fault object") Link: https://patch.msgid.link/r/336a9b6e44fe66a24199d3be777c405c85c98622.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 172fc8b19825a0f5884c38f2289188284e2d45ee) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 9388e8a2e42e..3bfc6df085cf 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -168,6 +168,8 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, } done += fault_size; } + if (rc) + break; } mutex_unlock(&fault->mutex); From 264ae90216391878dbc39fff8432d4d058981923 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:32:52 -0700 Subject: [PATCH 60/67] iommufd: Avoid partial fault group delivery in iommufd_fault_fops_read() JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 091ab6d70dc444f56ed14faedbcacfc979f4c613 Author: Nicolin Chen Date: Mon Jun 1 13:42:37 2026 -0700 iommufd: Avoid partial fault group delivery in iommufd_fault_fops_read() The cookie returned by xa_alloc() in iommufd_fault_fops_read() is per fault group, but the inner copy_to_user() runs per fault inside the group. If a copy fails mid-group, xa_erase clears the cookie and the group is restored to the deliver list, yet done is not rolled back. The function returns the partial byte count, with the successfully copied faults sitting at offsets below done carrying the now-erased cookie. The next read() then re-fetches the group, allocates a fresh cookie, and re-delivers every fault including the ones already copied; userspace sees duplicates carrying the new cookie, and a stale cookie that can never be responded to. Use a local group_done variable that tracks the per-group progress inside the inner loop, and only commit done = group_done after the inner loop has finished successfully. On a copy_to_user failure the outer break skips the commit, so done remains at its prior start-of-group baseline; the partial bytes already written past done are undefined to userspace per the read(2) contract, and the next read re-delivers the whole group atomically. Fixes: 07838f7fd529 ("iommufd: Add iommufd fault object") Link: https://patch.msgid.link/r/360cab4d4aeccb0bae275a970e2b3c340a71e0e0.1780343944.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 091ab6d70dc444f56ed14faedbcacfc979f4c613) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/eventq.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index 3bfc6df085cf..a934f02528b0 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -139,6 +139,8 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, mutex_lock(&fault->mutex); while ((group = iommufd_fault_deliver_fetch(fault))) { + size_t group_done = done; + if (done >= count || group->fault_count * fault_size > count - done) { iommufd_fault_deliver_restore(fault, group); @@ -160,16 +162,17 @@ static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, iommufd_compose_fault_message(&iopf->fault, &data, idev, group->cookie); - if (copy_to_user(buf + done, &data, fault_size)) { + if (copy_to_user(buf + group_done, &data, fault_size)) { xa_erase(&fault->response, group->cookie); iommufd_fault_deliver_restore(fault, group); rc = -EFAULT; break; } - done += fault_size; + group_done += fault_size; } if (rc) break; + done = group_done; } mutex_unlock(&fault->mutex); From da57eb8dbcb367898cba5d2dd0b03235a76ccdb5 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:33:12 -0700 Subject: [PATCH 61/67] iommufd: Destroy the pages content after detaching from dmabuf JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit f2d70dbd3dcefa8e3c380beff9c31f5f033a4221 Author: Jason Gunthorpe Date: Mon Jun 8 21:20:25 2026 -0300 iommufd: Destroy the pages content after detaching from dmabuf Sashiko points out this has gotten out of order, the mutex could still be in use through the dmabuf invalidation callbacks. Don't destroy any of the pages content until the dmabuf is fully detached. Fixes: 71db84a092c3 ("iommufd: Add DMABUF to iopt_pages") Signed-off-by: Jason Gunthorpe (cherry picked from commit f2d70dbd3dcefa8e3c380beff9c31f5f033a4221) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/pages.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index f863fea75b98..7c70d05c4a9b 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -1650,10 +1650,6 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); - mmdrop(pages->source_mm); - mutex_destroy(&pages->mutex); - put_task_struct(pages->source_task); - free_uid(pages->source_user); if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1663,6 +1659,10 @@ void iopt_release_pages(struct kref *kref) } else if (pages->type == IOPT_ADDRESS_FILE) { fput(pages->file); } + mmdrop(pages->source_mm); + mutex_destroy(&pages->mutex); + put_task_struct(pages->source_task); + free_uid(pages->source_user); kfree(pages); } From e60025058f4c7fdc02ddcf4e7fdfccafa2da2c81 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:33:25 -0700 Subject: [PATCH 62/67] iommufd: Clarify IOAS_MAP_FILE dma-buf support JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 298ab7e6f1637cec44163f52e84e2030ec16ed9d Author: Alex Mastro Date: Wed Jun 10 13:44:41 2026 -0700 iommufd: Clarify IOAS_MAP_FILE dma-buf support IOMMU_IOAS_MAP_FILE is documented as mapping a memfd, but the implementation first tries to resolve the fd as a dma-buf and has a special path for supported dma-buf exporters. In particular, VFIO PCI dma-bufs exported through VFIO_DEVICE_FEATURE_DMA_BUF can be mapped when they describe a single DMA range. Update the UAPI comment so userspace understands that certain kinds of dma-buf are supported in addition to memfd. Fixes: 44ebaa1744fd ("iommufd: Accept a DMABUF through IOMMU_IOAS_MAP_FILE") Link: https://patch.msgid.link/r/20260610-tmp-v1-1-b8ccbf557391@fb.com Signed-off-by: Alex Mastro Assisted-by: Codex:gpt-5.5-high Signed-off-by: Jason Gunthorpe (cherry picked from commit 298ab7e6f1637cec44163f52e84e2030ec16ed9d) Signed-off-by: Jerry Snitselaar --- include/uapi/linux/iommufd.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index e998dfbd6960..0425d452d41e 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -224,13 +224,17 @@ struct iommu_ioas_map { * @size: sizeof(struct iommu_ioas_map_file) * @flags: same as for iommu_ioas_map * @ioas_id: same as for iommu_ioas_map - * @fd: the memfd to map - * @start: byte offset from start of file to map from + * @fd: the memfd or supported dma-buf file to map + * @start: byte offset from start of the file to map from * @length: same as for iommu_ioas_map * @iova: same as for iommu_ioas_map * - * Set an IOVA mapping from a memfd file. All other arguments and semantics - * match those of IOMMU_IOAS_MAP. + * Set an IOVA mapping from a memfd file. On kernels with dma-buf support, + * supported dma-buf files may also be accepted. This is not a generic + * dma-buf import path; currently supported dma-bufs include single-range + * VFIO PCI dma-bufs exported through VFIO_DEVICE_FEATURE_DMA_BUF, and + * other dma-bufs may be rejected. All other arguments and semantics match + * those of IOMMU_IOAS_MAP. */ struct iommu_ioas_map_file { __u32 size; From 3d4615614ecbd77b9f16a49a2e44b9541bd771c4 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Tue, 21 Jul 2026 11:33:41 -0700 Subject: [PATCH 63/67] iommufd: Set upper bounds on cache invalidation entry_num and entry_len JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 4d70986002f2f3eaaed89124fb2522bded38b016 Author: Nicolin Chen Date: Wed Jun 3 14:26:53 2026 -0700 iommufd: Set upper bounds on cache invalidation entry_num and entry_len iommufd_hwpt_invalidate() takes a user-controlled entry_num and entry_len, each bounded only by U32_MAX. An entry_len beyond the kernel's struct size makes the copy helper verify the extra bytes are zero, scanning that excess in one uninterruptible pass; a multi-gigabyte value over zeroed user memory trips the soft-lockup watchdog. A large entry_num is the other half, driving the backend invalidation loop with no reschedule. The VT-d nested handler, for one, copies each entry and flushes caches per iteration, pinning the CPU on a non-preemptible kernel. Cap both in the ioctl. entry_len is held under PAGE_SIZE, above any request struct, and entry_num under 1 << 19, the order of a hardware invalidation queue and well beyond any real batch, bounding the per-call loop length. Fixes: 8c6eabae3807 ("iommufd: Add IOMMU_HWPT_INVALIDATE") Link: https://patch.msgid.link/r/447fa93663f7526eb361719e83fa8b649464483d.1780521606.git.nicolinc@nvidia.com Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Nicolin Chen Reviewed-by: Lu Baolu Signed-off-by: Jason Gunthorpe (cherry picked from commit 4d70986002f2f3eaaed89124fb2522bded38b016) Signed-off-by: Jerry Snitselaar --- drivers/iommu/iommufd/hw_pagetable.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c index fe789c2dc0c9..623cc608ca0c 100644 --- a/drivers/iommu/iommufd/hw_pagetable.c +++ b/drivers/iommu/iommufd/hw_pagetable.c @@ -489,6 +489,9 @@ int iommufd_hwpt_get_dirty_bitmap(struct iommufd_ucmd *ucmd) return rc; } +/* An arbitrary entry_num cap, far above any realistic invalidation batch */ +#define IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX (1U << 19) + int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd) { struct iommu_hwpt_invalidate *cmd = ucmd->cmd; @@ -507,7 +510,13 @@ int iommufd_hwpt_invalidate(struct iommufd_ucmd *ucmd) goto out; } - if (cmd->entry_num && (!cmd->data_uptr || !cmd->entry_len)) { + /* + * Bound entry_num and entry_len so a single call cannot pin the CPU; + * entry_len also caps the copy_struct_from_user() trailing-zero scan. + */ + if (cmd->entry_num && + (!cmd->data_uptr || !cmd->entry_len || cmd->entry_len > PAGE_SIZE || + cmd->entry_num > IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX)) { rc = -EINVAL; goto out; } From e3545fc32b1e0304063b451d4431694b36ba053e Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Wed, 22 Jul 2026 10:08:17 -0700 Subject: [PATCH 64/67] iommufd/selftest: Add boundary tests for veventq_depth JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit f25989c19028e8bf81e26e1133a99e3436c3afc2 Author: Nicolin Chen Date: Thu May 21 17:36:35 2026 -0700 iommufd/selftest: Add boundary tests for veventq_depth Test veventq_depth to cover a memory exhaustion vulnerability. Keep veventq_depth=2 for the existing callers. Link: https://patch.msgid.link/r/acfa370fa4e89e4626f71954bad7ad2bd64cf63b.1779408671.git.nicolinc@nvidia.com Reviewed-by: Jason Gunthorpe Signed-off-by: Nicolin Chen Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit f25989c19028e8bf81e26e1133a99e3436c3afc2) Signed-off-by: Jerry Snitselaar --- tools/testing/selftests/iommu/iommufd.c | 19 +++++++++++++++++-- .../selftests/iommu/iommufd_fail_nth.c | 2 +- tools/testing/selftests/iommu/iommufd_utils.h | 17 +++++++++-------- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index dadad277f4eb..dd102b6c3a13 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -2959,11 +2959,26 @@ TEST_F(iommufd_viommu, vdevice_alloc) test_err_mock_domain_replace(ENOENT, self->stdev_id, self->nested_hwpt_id); + /* Test depth lower and upper bounds (mirrors kernel cap) */ +#define VEVENTQ_MAX_DEPTH (1U << 19) + test_err_veventq_alloc(EINVAL, viommu_id, + IOMMU_VEVENTQ_TYPE_SELFTEST, 0, NULL, + NULL); + test_err_veventq_alloc(EINVAL, viommu_id, + IOMMU_VEVENTQ_TYPE_SELFTEST, + VEVENTQ_MAX_DEPTH + 1, NULL, NULL); + test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST, + VEVENTQ_MAX_DEPTH, &veventq_id, + &veventq_fd); + close(veventq_fd); + test_ioctl_destroy(veventq_id); + /* Allocate a vEVENTQ with veventq_depth=2 */ test_cmd_veventq_alloc(viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST, - &veventq_id, &veventq_fd); + 2, &veventq_id, &veventq_fd); test_err_veventq_alloc(EEXIST, viommu_id, - IOMMU_VEVENTQ_TYPE_SELFTEST, NULL, NULL); + IOMMU_VEVENTQ_TYPE_SELFTEST, 2, NULL, + NULL); /* Set vdev_id to 0x99, unset it, and set to 0x88 */ test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); test_cmd_mock_domain_replace(self->stdev_id, diff --git a/tools/testing/selftests/iommu/iommufd_fail_nth.c b/tools/testing/selftests/iommu/iommufd_fail_nth.c index 45c14323a618..25495d8dceb3 100644 --- a/tools/testing/selftests/iommu/iommufd_fail_nth.c +++ b/tools/testing/selftests/iommu/iommufd_fail_nth.c @@ -712,7 +712,7 @@ TEST_FAIL_NTH(basic_fail_nth, device) return -1; if (_test_cmd_veventq_alloc(self->fd, viommu_id, - IOMMU_VEVENTQ_TYPE_SELFTEST, &veventq_id, + IOMMU_VEVENTQ_TYPE_SELFTEST, 2, &veventq_id, &veventq_fd)) return -1; close(veventq_fd); diff --git a/tools/testing/selftests/iommu/iommufd_utils.h b/tools/testing/selftests/iommu/iommufd_utils.h index 0a0ff6f7926d..bad7175efd52 100644 --- a/tools/testing/selftests/iommu/iommufd_utils.h +++ b/tools/testing/selftests/iommu/iommufd_utils.h @@ -1060,12 +1060,13 @@ static int _test_cmd_hw_queue_alloc(int fd, __u32 viommu_id, __u32 type, base_addr, len, out_qid)) static int _test_cmd_veventq_alloc(int fd, __u32 viommu_id, __u32 type, - __u32 *veventq_id, __u32 *veventq_fd) + __u32 depth, __u32 *veventq_id, + __u32 *veventq_fd) { struct iommu_veventq_alloc cmd = { .size = sizeof(cmd), .type = type, - .veventq_depth = 2, + .veventq_depth = depth, .viommu_id = viommu_id, }; int ret; @@ -1080,13 +1081,13 @@ static int _test_cmd_veventq_alloc(int fd, __u32 viommu_id, __u32 type, return 0; } -#define test_cmd_veventq_alloc(viommu_id, type, veventq_id, veventq_fd) \ - ASSERT_EQ(0, _test_cmd_veventq_alloc(self->fd, viommu_id, type, \ +#define test_cmd_veventq_alloc(viommu_id, type, depth, veventq_id, veventq_fd) \ + ASSERT_EQ(0, _test_cmd_veventq_alloc(self->fd, viommu_id, type, depth, \ veventq_id, veventq_fd)) -#define test_err_veventq_alloc(_errno, viommu_id, type, veventq_id, \ - veventq_fd) \ - EXPECT_ERRNO(_errno, \ - _test_cmd_veventq_alloc(self->fd, viommu_id, type, \ +#define test_err_veventq_alloc(_errno, viommu_id, type, depth, veventq_id, \ + veventq_fd) \ + EXPECT_ERRNO(_errno, \ + _test_cmd_veventq_alloc(self->fd, viommu_id, type, depth, \ veventq_id, veventq_fd)) static int _test_cmd_trigger_vevents(int fd, __u32 dev_id, __u32 nvevents) From a53b0c5986f05ca57a09fe3355db66089abb8097 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Wed, 22 Jul 2026 10:08:33 -0700 Subject: [PATCH 65/67] iommufd/selftest: Cover invalid read counts on vEVENTQ FD JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 61d525d96ade3367ce65e340b33aa2f1fd5a3602 Author: Nicolin Chen Date: Mon Jun 1 13:42:38 2026 -0700 iommufd/selftest: Cover invalid read counts on vEVENTQ FD The vEVENTQ file descriptor must reject reads whose buffer cannot hold even one event record. Add selftest coverage that exercises both the empty-queue path (the upfront size check) and the non-empty path (the in-loop check that fires only after an event is fetched). For iommufd_veventq_fops_read(): - count == 0 and count < sizeof(header) on an empty vEVENTQ both return -EINVAL. - count == 0 and count == sizeof(header) on a non-empty vEVENTQ (event has trailing payload) both return -EINVAL. Link: https://patch.msgid.link/r/7bcd153d306f2cf04c094c728c0ebe146855072a.1780343944.git.nicolinc@nvidia.com Assisted-by: Claude:claude-opus-4-7 Signed-off-by: Nicolin Chen Reviewed-by: Pranjal Shrivastava Reviewed-by: Kevin Tian Signed-off-by: Jason Gunthorpe (cherry picked from commit 61d525d96ade3367ce65e340b33aa2f1fd5a3602) Signed-off-by: Jerry Snitselaar --- tools/testing/selftests/iommu/iommufd.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index dd102b6c3a13..c8448dd34d1a 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -2953,6 +2953,8 @@ TEST_F(iommufd_viommu, vdevice_alloc) uint32_t veventq_id; uint32_t veventq_fd; int prev_seq = -1; + size_t hdr_size = sizeof(struct iommufd_vevent_header); + char vbuf[64]; if (dev_id) { /* Must allocate vdevice before attaching to a nested hwpt */ @@ -2979,11 +2981,26 @@ TEST_F(iommufd_viommu, vdevice_alloc) test_err_veventq_alloc(EEXIST, viommu_id, IOMMU_VEVENTQ_TYPE_SELFTEST, 2, NULL, NULL); + + /* Invalid read counts on an empty vEVENTQ */ + ASSERT_EQ(-1, read(veventq_fd, vbuf, 0)); + ASSERT_EQ(EINVAL, errno); + ASSERT_EQ(-1, read(veventq_fd, vbuf, hdr_size - 1)); + ASSERT_EQ(EINVAL, errno); + /* Set vdev_id to 0x99, unset it, and set to 0x88 */ test_cmd_vdevice_alloc(viommu_id, dev_id, 0x99, &vdev_id); test_cmd_mock_domain_replace(self->stdev_id, self->nested_hwpt_id); test_cmd_trigger_vevents(dev_id, 1); + + /* Invalid read counts on a non-empty vEVENTQ */ + ASSERT_EQ(-1, read(veventq_fd, vbuf, 0)); + ASSERT_EQ(EINVAL, errno); + /* header fits but the event's payload doesn't */ + ASSERT_EQ(-1, read(veventq_fd, vbuf, hdr_size)); + ASSERT_EQ(EINVAL, errno); + test_cmd_read_vevents(veventq_fd, 1, 0x99, &prev_seq); test_err_vdevice_alloc(EEXIST, viommu_id, dev_id, 0x99, &vdev_id); From 94fd27fba786bd68a3e89a679dab2d85146bf44b Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Wed, 22 Jul 2026 10:09:05 -0700 Subject: [PATCH 66/67] iommufd/selftest: Add invalidation entry_num and entry_len boundary tests JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 5623c8cb81365f845f318135979d25a4b6671900 Author: Nicolin Chen Date: Wed Jun 3 14:26:54 2026 -0700 iommufd/selftest: Add invalidation entry_num and entry_len boundary tests Test that the cache invalidation ioctl rejects an oversized entry_len and an oversized entry_num, covering the CPU soft-lockup paths the caps close. Link: https://patch.msgid.link/r/9ae78ed8e64afbb2f2df27d03466380061adf7d9.1780521606.git.nicolinc@nvidia.com Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Nicolin Chen Reviewed-by: Lu Baolu Signed-off-by: Jason Gunthorpe (cherry picked from commit 5623c8cb81365f845f318135979d25a4b6671900) Signed-off-by: Jerry Snitselaar --- tools/testing/selftests/iommu/iommufd.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/testing/selftests/iommu/iommufd.c b/tools/testing/selftests/iommu/iommufd.c index c8448dd34d1a..8a823f18c70d 100644 --- a/tools/testing/selftests/iommu/iommufd.c +++ b/tools/testing/selftests/iommu/iommufd.c @@ -556,6 +556,21 @@ TEST_F(iommufd_ioas, alloc_hwpt_nested) 1, &num_inv); assert(!num_inv); + /* Negative test: entry_len is bounded by PAGE_SIZE */ + num_inv = 1; + test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, + IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, + PAGE_SIZE + 1, &num_inv); + assert(!num_inv); + + /* Negative test: entry_num is bounded */ +#define IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX (1U << 19) + num_inv = IOMMU_HWPT_INVALIDATE_ENTRY_NUM_MAX + 1; + test_err_hwpt_invalidate(EINVAL, nested_hwpt_id[0], inv_reqs, + IOMMU_HWPT_INVALIDATE_DATA_SELFTEST, + sizeof(*inv_reqs), &num_inv); + assert(!num_inv); + /* Negative test: invalid flag is passed */ num_inv = 1; inv_reqs[0].flags = 0xffffffff; From 72660b1c2b85050fd95f3e37dcde048299f5d3c4 Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Wed, 22 Jul 2026 11:35:42 -0700 Subject: [PATCH 67/67] iommu/vt-d: Avoid WARNING in sva unbind path JIRA: https://issues.redhat.com/browse/RHEL-213791 Upstream-Status: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git commit 534b5f98ab7319d8004bbc7dab6481462243e883 Author: Lu Baolu Date: Thu Jun 4 14:03:06 2026 +0800 iommu/vt-d: Avoid WARNING in sva unbind path The Intel IOMMU driver allows SVA on devices even if they do not support PCI/PRI. Commit 39c20c4e83b9 ("iommu/vt-d: Only handle IOPF for SVA when PRI is supported") modified the SVA bind path to allow this configuration by skipping IOPF enablement when PRI is missing. However, it failed to update the unbind path. This creates an imbalance: the unbind path attempts to disable IOPF for a device that never had it enabled, triggering a WARNING in intel_iommu_disable_iopf(): WARNING: drivers/iommu/intel/iommu.c:3475 at intel_iommu_disable_iopf+0x4f/0x90d Call Trace: blocking_domain_set_dev_pasid+0x50/0x70 iommu_detach_device_pasid+0x89/0xc0 iommu_sva_unbind_device+0x73/0x150 xe_vm_close_and_put+0x4d2/0x1200 [xe] Fix this by bypassing IOPF operations for SVA domains on non-PRI hardware in both the bind and unbind paths. Fixes: 39c20c4e83b9 ("iommu/vt-d: Only handle IOPF for SVA when PRI is supported") Cc: stable@vger.kernel.org Reported-by: Nareshkumar Gollakoti Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20260519052917.3729796-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel (cherry picked from commit 534b5f98ab7319d8004bbc7dab6481462243e883) Signed-off-by: Jerry Snitselaar --- drivers/iommu/intel/iommu.h | 11 +++++++++++ drivers/iommu/intel/svm.c | 12 ++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 599913fb65d5..b3a8686059d9 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1265,18 +1265,29 @@ void intel_iommu_disable_iopf(struct device *dev); static inline int iopf_for_domain_set(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return 0; + /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return 0; + return intel_iommu_enable_iopf(dev); } static inline void iopf_for_domain_remove(struct iommu_domain *domain, struct device *dev) { + struct device_domain_info *info = dev_iommu_priv_get(dev); + if (!domain || !domain->iopf_handler) return; + if (domain->type == IOMMU_DOMAIN_SVA && !info->pri_supported) + return; + intel_iommu_disable_iopf(dev); } diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index b1389df36ef8..71de7947971f 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -164,12 +164,9 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, if (IS_ERR(dev_pasid)) return PTR_ERR(dev_pasid); - /* SVA with non-IOMMU/PRI IOPF handling is allowed. */ - if (info->pri_supported) { - ret = iopf_for_domain_replace(domain, old, dev); - if (ret) - goto out_remove_dev_pasid; - } + ret = iopf_for_domain_replace(domain, old, dev); + if (ret) + goto out_remove_dev_pasid; /* Setup the pasid table: */ sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0; @@ -184,8 +181,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, return 0; out_unwind_iopf: - if (info->pri_supported) - iopf_for_domain_replace(old, domain, dev); + iopf_for_domain_replace(old, domain, dev); out_remove_dev_pasid: domain_remove_dev_pasid(domain, dev, pasid); return ret;