From da6d997ac556479c112554ab5d95cbd04683eb11 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:05 +0300 Subject: [PATCH 1/9] ntb: Store original DMA address for future release The DMA API requires that dma_free_attrs receive the exact dma_handle originally returned by the allocation function. Do not modify it. Fixes: fc5d1829f9bf ("NTB: transport: Try harder to alloc an aligned MW buffer") Signed-off-by: Leon Romanovsky Reviewed-by: Dave Jiang Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-1-8dbac75cd501@nvidia.com --- drivers/ntb/ntb_transport.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 7cabc82305d6..771eb7d9f435 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -223,6 +223,7 @@ struct ntb_transport_mw { void *alloc_addr; void *virt_addr; dma_addr_t dma_addr; + dma_addr_t original_dma_addr; }; struct ntb_transport_client_dev { @@ -766,7 +767,7 @@ static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); dma_free_coherent(dma_dev, mw->alloc_size, - mw->alloc_addr, mw->dma_addr); + mw->alloc_addr, mw->original_dma_addr); mw->xlat_size = 0; mw->buff_size = 0; mw->alloc_size = 0; @@ -802,6 +803,7 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, return -ENOMEM; } virt_addr = alloc_addr; + mw->original_dma_addr = dma_addr; /* * we must ensure that the memory address allocated is BAR size From 9d625aa2ebd445868955719e0abcf695b43f9318 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:06 +0300 Subject: [PATCH 2/9] ntb: Use consistent DMA attributes when freeing DMA mappings The counterpart of dma_alloc_attrs() is dma_free_attrs(), which must receive the same DMA attributes used during allocation. The code previously used dma_free_coherent(), which does not accept or apply any DMA attributes. Fixes: 061a785a114f ("ntb: Force physically contiguous allocation of rx ring buffers") Signed-off-by: Leon Romanovsky Reviewed-by: Dave Jiang Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-2-8dbac75cd501@nvidia.com --- drivers/ntb/ntb_transport.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 771eb7d9f435..f59f926d4bfa 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -766,8 +766,8 @@ static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw) return; ntb_mw_clear_trans(nt->ndev, PIDX, num_mw); - dma_free_coherent(dma_dev, mw->alloc_size, - mw->alloc_addr, mw->original_dma_addr); + dma_free_attrs(dma_dev, mw->alloc_size, mw->alloc_addr, + mw->original_dma_addr, DMA_ATTR_FORCE_CONTIGUOUS); mw->xlat_size = 0; mw->buff_size = 0; mw->alloc_size = 0; @@ -828,8 +828,8 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw, return 0; err: - dma_free_coherent(ntb_dev, mw->alloc_size, alloc_addr, dma_addr); - + dma_free_attrs(ntb_dev, mw->alloc_size, alloc_addr, dma_addr, + DMA_ATTR_FORCE_CONTIGUOUS); return rc; } From 5da80b8fb38006c7e79b06cba711c28219f17ab2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:07 +0300 Subject: [PATCH 3/9] dma-debug: Remove unused DMA attribute parameter debug_dma_alloc_pages() always receives a DMA attribute value of 0, because dma_alloc_pages() never receives any attributes from its callers. As preparation for upcoming patches, remove this unused attribute from the debug routine. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-3-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 5 ++--- kernel/dma/debug.h | 6 ++---- kernel/dma/mapping.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 1a725edbbbf6..3b53495337f5 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1567,8 +1567,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs) + dma_addr_t dma_addr) { struct dma_debug_entry *entry; @@ -1586,7 +1585,7 @@ void debug_dma_alloc_pages(struct device *dev, struct page *page, entry->dev_addr = dma_addr; entry->direction = direction; - add_dma_entry(entry, attrs); + add_dma_entry(entry, 0); } void debug_dma_free_pages(struct device *dev, struct page *page, diff --git a/kernel/dma/debug.h b/kernel/dma/debug.h index da7be0bddcf6..24b8610850fb 100644 --- a/kernel/dma/debug.h +++ b/kernel/dma/debug.h @@ -47,8 +47,7 @@ extern void debug_dma_sync_sg_for_device(struct device *dev, int nelems, int direction); extern void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs); + dma_addr_t dma_addr); extern void debug_dma_free_pages(struct device *dev, struct page *page, size_t size, int direction, dma_addr_t dma_addr); @@ -113,8 +112,7 @@ static inline void debug_dma_sync_sg_for_device(struct device *dev, static inline void debug_dma_alloc_pages(struct device *dev, struct page *page, size_t size, int direction, - dma_addr_t dma_addr, - unsigned long attrs) + dma_addr_t dma_addr) { } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 23ed8eb9233e..6cbefbd4158c 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -733,7 +733,7 @@ struct page *dma_alloc_pages(struct device *dev, size_t size, if (page) { trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle, size, dir, gfp, 0); - debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0); + debug_dma_alloc_pages(dev, page, size, dir, *dma_handle); } else { trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); } From 04d492ab964a6256a93ed11cdf99b126281e793a Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:08 +0300 Subject: [PATCH 4/9] dma-debug: Record DMA attributes in debug entry To enable reliable comparison of DMA attributes between map and unmap operations, store the attribute value in dma_debug_entry. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-4-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 48 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 3b53495337f5..f07e6a1e9fba 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -63,7 +63,7 @@ enum map_err_types { * @sg_mapped_ents: 'mapped_ents' from dma_map_sg * @paddr: physical start address of the mapping * @map_err_type: track whether dma_mapping_error() was checked - * @is_cache_clean: driver promises not to write to buffer while mapped + * @attrs: dma attributes * @stack_len: number of backtrace entries in @stack_entries * @stack_entries: stack of backtrace history */ @@ -78,7 +78,7 @@ struct dma_debug_entry { int sg_mapped_ents; phys_addr_t paddr; enum map_err_types map_err_type; - bool is_cache_clean; + unsigned long attrs; #ifdef CONFIG_STACKTRACE unsigned int stack_len; unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; @@ -478,6 +478,9 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, bool *overlap_cache_clean) { phys_addr_t cln = to_cacheline_number(entry); + bool is_cache_clean = entry->attrs & + (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT); unsigned long flags; int rc; @@ -495,12 +498,15 @@ static int active_cacheline_insert(struct dma_debug_entry *entry, if (rc == -EEXIST) { struct dma_debug_entry *existing; - active_cacheline_inc_overlap(cln, entry->is_cache_clean); + active_cacheline_inc_overlap(cln, is_cache_clean); existing = radix_tree_lookup(&dma_active_cacheline, cln); /* A lookup failure here after we got -EEXIST is unexpected. */ WARN_ON(!existing); if (existing) - *overlap_cache_clean = existing->is_cache_clean; + *overlap_cache_clean = + existing->attrs & + (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT); } spin_unlock_irqrestore(&radix_lock, flags); @@ -544,12 +550,13 @@ void debug_dma_dump_mappings(struct device *dev) if (!dev || dev == entry->dev) { cln = to_cacheline_number(entry); dev_info(entry->dev, - "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", + "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n", type2name[entry->type], idx, &entry->paddr, entry->dev_addr, entry->size, &cln, dir2name[entry->direction], - maperr2str[entry->map_err_type]); + maperr2str[entry->map_err_type], + entry->attrs); } } spin_unlock_irqrestore(&bucket->lock, flags); @@ -575,14 +582,15 @@ static int dump_show(struct seq_file *seq, void *v) list_for_each_entry(entry, &bucket->list, list) { cln = to_cacheline_number(entry); seq_printf(seq, - "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n", + "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n", dev_driver_string(entry->dev), dev_name(entry->dev), type2name[entry->type], idx, &entry->paddr, entry->dev_addr, entry->size, &cln, dir2name[entry->direction], - maperr2str[entry->map_err_type]); + maperr2str[entry->map_err_type], + entry->attrs); } spin_unlock_irqrestore(&bucket->lock, flags); } @@ -594,16 +602,14 @@ DEFINE_SHOW_ATTRIBUTE(dump); * Wrapper function for adding an entry to the hash. * This function takes care of locking itself. */ -static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) +static void add_dma_entry(struct dma_debug_entry *entry) { + unsigned long attrs = entry->attrs; bool overlap_cache_clean; struct hash_bucket *bucket; unsigned long flags; int rc; - entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | - DMA_ATTR_REQUIRE_COHERENT); - bucket = get_hash_bucket(entry, &flags); hash_bucket_add(bucket, entry); put_hash_bucket(bucket, flags); @@ -612,9 +618,10 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) if (rc == -ENOMEM) { pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); global_disable = true; - } else if (rc == -EEXIST && - !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - !(entry->is_cache_clean && overlap_cache_clean) && + } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + !(attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES | + DMA_ATTR_REQUIRE_COHERENT) && + overlap_cache_clean) && dma_get_cache_alignment() >= L1_CACHE_BYTES && !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && is_swiotlb_active(entry->dev))) { @@ -1250,6 +1257,7 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, entry->size = size; entry->direction = direction; entry->map_err_type = MAP_ERR_NOT_CHECKED; + entry->attrs = attrs; if (!(attrs & DMA_ATTR_MMIO)) { check_for_stack(dev, phys); @@ -1258,7 +1266,7 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, check_for_illegal_area(dev, phys_to_virt(phys), size); } - add_dma_entry(entry, attrs); + add_dma_entry(entry); } void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) @@ -1345,10 +1353,11 @@ void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, entry->direction = direction; entry->sg_call_ents = nents; entry->sg_mapped_ents = mapped_ents; + entry->attrs = attrs; check_sg_segment(dev, s); - add_dma_entry(entry, attrs); + add_dma_entry(entry); } } @@ -1440,8 +1449,9 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size, entry->size = size; entry->dev_addr = dma_addr; entry->direction = DMA_BIDIRECTIONAL; + entry->attrs = attrs; - add_dma_entry(entry, attrs); + add_dma_entry(entry); } void debug_dma_free_coherent(struct device *dev, size_t size, @@ -1585,7 +1595,7 @@ void debug_dma_alloc_pages(struct device *dev, struct page *page, entry->dev_addr = dma_addr; entry->direction = direction; - add_dma_entry(entry, 0); + add_dma_entry(entry); } void debug_dma_free_pages(struct device *dev, struct page *page, From c8411b1d1e524cbe4a12aacad7bf2163fb2be062 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:09 +0300 Subject: [PATCH 5/9] dma-debug: Feed DMA attribute for unmapping flows too There are multiple unmapping flows which didn't provide DMA attributes, which limited DMA debug code to compare the mapping and unmapping attributes. Let's fix it. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-5-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 13 ++++++++----- kernel/dma/debug.h | 19 +++++++++++-------- kernel/dma/mapping.c | 8 ++++---- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index f07e6a1e9fba..3dfed51c3d9a 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1307,8 +1307,8 @@ void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) } EXPORT_SYMBOL(debug_dma_mapping_error); -void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, - size_t size, int direction) +void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, + int direction, unsigned long attrs) { struct dma_debug_entry ref = { .type = dma_debug_phy, @@ -1316,6 +1316,7 @@ void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, .dev_addr = dma_addr, .size = size, .direction = direction, + .attrs = attrs, }; if (unlikely(dma_debug_disabled())) @@ -1381,7 +1382,7 @@ static int get_nr_mapped_entries(struct device *dev, } void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nelems, int dir) + int nelems, int dir, unsigned long attrs) { struct scatterlist *s; int mapped_ents = 0, i; @@ -1399,6 +1400,7 @@ void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, .size = sg_dma_len(s), .direction = dir, .sg_call_ents = nelems, + .attrs = attrs, }; if (mapped_ents && i >= mapped_ents) @@ -1454,8 +1456,8 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size, add_dma_entry(entry); } -void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t dma_addr) +void debug_dma_free_coherent(struct device *dev, size_t size, void *virt, + dma_addr_t dma_addr, unsigned long attrs) { struct dma_debug_entry ref = { .type = dma_debug_coherent, @@ -1463,6 +1465,7 @@ void debug_dma_free_coherent(struct device *dev, size_t size, .dev_addr = dma_addr, .size = size, .direction = DMA_BIDIRECTIONAL, + .attrs = attrs, }; /* handle vmalloc and linear addresses */ diff --git a/kernel/dma/debug.h b/kernel/dma/debug.h index 24b8610850fb..13e384633c32 100644 --- a/kernel/dma/debug.h +++ b/kernel/dma/debug.h @@ -14,21 +14,22 @@ extern void debug_dma_map_phys(struct device *dev, phys_addr_t phys, unsigned long attrs); extern void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, - size_t size, int direction); + size_t size, int direction, + unsigned long attrs); extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, int mapped_ents, int direction, unsigned long attrs); extern void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, - int nelems, int dir); + int nelems, int dir, unsigned long attrs); extern void debug_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t dma_addr, void *virt, unsigned long attrs); -extern void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t addr); +extern void debug_dma_free_coherent(struct device *dev, size_t size, void *virt, + dma_addr_t addr, unsigned long attrs); extern void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, @@ -59,7 +60,8 @@ static inline void debug_dma_map_phys(struct device *dev, phys_addr_t phys, } static inline void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr, - size_t size, int direction) + size_t size, int direction, + unsigned long attrs) { } @@ -70,8 +72,8 @@ static inline void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, } static inline void debug_dma_unmap_sg(struct device *dev, - struct scatterlist *sglist, - int nelems, int dir) + struct scatterlist *sglist, int nelems, + int dir, unsigned long attrs) { } @@ -82,7 +84,8 @@ static inline void debug_dma_alloc_coherent(struct device *dev, size_t size, } static inline void debug_dma_free_coherent(struct device *dev, size_t size, - void *virt, dma_addr_t addr) + void *virt, dma_addr_t addr, + unsigned long attrs) { } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 6cbefbd4158c..f010b3cc0ece 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -225,7 +225,7 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, else if (ops->unmap_phys) ops->unmap_phys(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); - debug_dma_unmap_phys(dev, addr, size, dir); + debug_dma_unmap_phys(dev, addr, size, dir, attrs); } EXPORT_SYMBOL_GPL(dma_unmap_phys); @@ -351,7 +351,7 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, BUG_ON(!valid_dma_direction(dir)); trace_dma_unmap_sg(dev, sg, nents, dir, attrs); - debug_dma_unmap_sg(dev, sg, nents, dir); + debug_dma_unmap_sg(dev, sg, nents, dir, attrs); if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); @@ -693,7 +693,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, if (!cpu_addr) return; - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); + debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs); if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle)) dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); else if (use_dma_iommu(dev)) @@ -840,7 +840,7 @@ void dma_free_noncontiguous(struct device *dev, size_t size, struct sg_table *sgt, enum dma_data_direction dir) { trace_dma_free_sgt(dev, sgt, size, dir); - debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); + debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0); if (use_dma_iommu(dev)) iommu_dma_free_noncontiguous(dev, size, sgt, dir); From a34bd60bdaebb1db4631cad45ff87dcabf4fc962 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Fri, 1 May 2026 09:35:10 +0300 Subject: [PATCH 6/9] dma-debug: Ensure mappings are created and released with matching attributes The DMA API expects that callers use the same attributes when mapping and unmapping. Add tracking to verify this and catch mismatches. Signed-off-by: Leon Romanovsky Reviewed-by: Samiullah Khawaja Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260501-dma-attrs-debug-v2-6-8dbac75cd501@nvidia.com --- kernel/dma/debug.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 3dfed51c3d9a..c38efc1ac8d6 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -1074,6 +1074,29 @@ static void check_unmap(struct dma_debug_entry *ref) type2name[entry->type]); } + /* + * This may be no bug in reality - but DMA API still expects + * that entry is unmapped with same attributes as it was mapped. + * + * DMA_ATTR_UNMAP_VALID lists the attributes that must be identical + * between map and unmap. Any attribute outside this set (e.g. + * DMA_ATTR_NO_WARN, DMA_ATTR_SKIP_CPU_SYNC) is allowed to differ. + */ +#define DMA_ATTR_UNMAP_VALID \ + (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_FORCE_CONTIGUOUS | \ + DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_PRIVILEGED | \ + DMA_ATTR_CC_SHARED) + if ((ref->attrs & DMA_ATTR_UNMAP_VALID) != + (entry->attrs & DMA_ATTR_UNMAP_VALID)) { + err_printk(ref->dev, entry, + "device driver frees " + "DMA memory with different attributes " + "[device address=0x%016llx] [size=%llu bytes] " + "[mapped with 0x%lx] [unmapped with 0x%lx]\n", + ref->dev_addr, ref->size, entry->attrs, ref->attrs); + } +#undef DMA_ATTR_UNMAP_VALID + hash_bucket_del(entry); put_hash_bucket(bucket, flags); From 7e6ace2535d032c908e4d8747d9a7952617c001a Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 May 2026 16:55:09 +0800 Subject: [PATCH 7/9] dma-contiguous: add kconfig option to setup numa cma area if not configured explicitly There was a report on a multi-numa-nodes arm64 server that when IOMMU is disabled, the dma_alloc_coherent() function always returns memory from node 0 even for devices attaching to other nodes, while they can get local dma memory when IOMMU is on with the same API. The reason is, when IOMMU is disabled, the dma_alloc_coherent() will go the direct way and call dma_alloc_contiguous(). The system doesn't have any explicit cma setting (like per-numa cma), and only has a default 64MB cma reserved area (on node 0), where kernel will try first to allocate memory from. Robin Murphy suggested to setup pernuma cma or disable cma, which did solve the issue. While there is still concern that for customers which don't have much kernel knowledge, they could still suffer from this silently as some architectures enable cma area by default (not an issue for X86 though, which set CONFIG_CMA_SIZE_MBYTES to 0 by default) for most Linux distributions. One thought is to follow the current cma reserving policy for platform with 'CONFIG_DMA_NUMA_CMA=y', that if the numa cma (either the 'numa cma' or 'cma pernuma' method) is not explicitly configured, and the platform really has multiple NUMA nodes, set it up according to size of default 'dma_contiguous_default_area'. This way, the default behavior of platform with one NUMA node is kept unchanged (say embedded/small devices don't need to allocate extra memory), while the general dma locality is improved. Add a new bool kernel config CONFIG_CMA_SIZE_PERNUMA to control whether to enable it. Even when the config is enabled, user can still disable it by kernel-cmdline setting like "numa_cma=0:0" or "cma_pernuma=0". Reported-by: Changrong Chen Suggested-by: Ying Huang Suggested-by: Robin Murphy Signed-off-by: Feng Tang Link: https://lore.kernel.org/r/20260512085509.83002-1-feng.tang@linux.alibaba.com Link: https://lore.kernel.org/all/20260520222742.GA1607511@ax162/ [mszyprow: squashed changes from both links, added __initdata attribute to the numa_cma_configured variable] Signed-off-by: Marek Szyprowski --- kernel/dma/Kconfig | 10 ++++++++++ kernel/dma/contiguous.c | 12 ++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index bfef21b4a9ae..c9fa0a922cba 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -181,6 +181,16 @@ config DMA_NUMA_CMA or set the node id and its size of CMA by specifying "numa_cma= :size[,:size]" on the kernel's command line. +config CMA_SIZE_PERNUMA + bool "Default CMA area per NUMA node" + depends on DMA_NUMA_CMA + default y + help + On systems with more than one NUMA node, the selected CMA + area size will be also allocated on each additional node, + so that most devices may have benefit from better DMA + locality without an explicit command-line opt-in. + comment "Default contiguous memory area size:" config CMA_SIZE_MBYTES diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 03f52bd17120..799f6e9c88bd 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -136,6 +136,7 @@ static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata; static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES]; static phys_addr_t pernuma_size_bytes __initdata; +static bool numa_cma_configured __initdata; static int __init early_numa_cma(char *p) { @@ -164,6 +165,7 @@ static int __init early_numa_cma(char *p) break; } + numa_cma_configured = true; return 0; } early_param("numa_cma", early_numa_cma); @@ -171,6 +173,7 @@ early_param("numa_cma", early_numa_cma); static int __init early_cma_pernuma(char *p) { pernuma_size_bytes = memparse(p, &p); + numa_cma_configured = true; return 0; } early_param("cma_pernuma", early_cma_pernuma); @@ -199,6 +202,11 @@ static void __init dma_numa_cma_reserve(void) { int nid; + if (IS_ENABLED(CONFIG_CMA_SIZE_PERNUMA) && + !numa_cma_configured && dma_contiguous_default_area && + nr_online_nodes > 1) + pernuma_size_bytes = cma_get_size(dma_contiguous_default_area); + for_each_node(nid) { int ret; char name[CMA_MAX_NAME]; @@ -255,8 +263,6 @@ void __init dma_contiguous_reserve(phys_addr_t limit) phys_addr_t selected_limit = limit; bool fixed = false; - dma_numa_cma_reserve(); - pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit); if (size_cmdline != -1) { @@ -312,6 +318,8 @@ void __init dma_contiguous_reserve(phys_addr_t limit) if (ret) pr_warn("Couldn't queue default CMA region for heap creation."); } + + dma_numa_cma_reserve(); } void __weak From d5cae2261b86913e602452ce4a07e6aefc0f603b Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Mon, 25 May 2026 09:51:11 +0800 Subject: [PATCH 8/9] dma-contiguous: simplify numa cma area handling Currently, there are 2 kernel cmdline ways to setup numa cma area: "cma_pernuma=" and "numa_cma=", and there are 2 cma arrays as well, while they have no difference technically. Robin suggested to cleanup the code and only use one array [1], as "the apparent intent that users only want one _or_ the other". Simplify the code by only using one array to save the numa cma area. And in rare case that a user really setup the 2 cmdline parameters at the same time, let the per-node specific size setting 'numa_cma=' take priority over the global numa cma setting. Link[1]: https://lore.kernel.org/lkml/43c5301c-fe6a-41e4-9482-ccfc7b62f2a7@arm.com/ Suggested-by: Robin Murphy Signed-off-by: Feng Tang Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260525015111.6267-1-feng.tang@linux.alibaba.com --- .../admin-guide/kernel-parameters.txt | 4 ++ kernel/dma/Kconfig | 4 +- kernel/dma/contiguous.c | 46 +++++-------------- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 4d0f545fb3ec..8e717a587870 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -895,6 +895,10 @@ Kernel parameters contiguous memory allocations. It will reserve CMA area for the specified node. + If it is setup together with upper 'cmd_pernuma=' + (unlikely), its size setting takes priority for the + specified numa nodes. + With numa CMA enabled, DMA users on node nid will first try to allocate buffer from the numa area which is located in node nid, if the allocation fails, diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index c9fa0a922cba..0a4ba21a57a7 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -179,7 +179,9 @@ config DMA_NUMA_CMA You can set the size of pernuma CMA by specifying "cma_pernuma=size" or set the node id and its size of CMA by specifying "numa_cma= - :size[,:size]" on the kernel's command line. + :size[,:size]" on the kernel's command line. And in + rare case that the above 2 are both setup, then the "numa_cma=" + takes priority for the specified numa nodes. config CMA_SIZE_PERNUMA bool "Default CMA area per NUMA node" diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 799f6e9c88bd..f754079a287d 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -134,7 +134,6 @@ EXPORT_SYMBOL_GPL(dev_get_cma_area); static struct cma *dma_contiguous_numa_area[MAX_NUMNODES]; static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata; -static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES]; static phys_addr_t pernuma_size_bytes __initdata; static bool numa_cma_configured __initdata; @@ -208,7 +207,7 @@ static void __init dma_numa_cma_reserve(void) pernuma_size_bytes = cma_get_size(dma_contiguous_default_area); for_each_node(nid) { - int ret; + int size, ret; char name[CMA_MAX_NAME]; struct cma **cma; @@ -218,27 +217,17 @@ static void __init dma_numa_cma_reserve(void) continue; } - if (pernuma_size_bytes) { + /* per-node numa setting has the priority */ + size = numa_cma_size[nid] ?: pernuma_size_bytes; + if (!size) + continue; - cma = &dma_contiguous_pernuma_area[nid]; - snprintf(name, sizeof(name), "pernuma%d", nid); - ret = cma_declare_contiguous_nid(0, pernuma_size_bytes, 0, 0, - 0, false, name, cma, nid); - if (ret) - pr_warn("%s: reservation failed: err %d, node %d", __func__, - ret, nid); - } - - if (numa_cma_size[nid]) { - - cma = &dma_contiguous_numa_area[nid]; - snprintf(name, sizeof(name), "numa%d", nid); - ret = cma_declare_contiguous_nid(0, numa_cma_size[nid], 0, 0, 0, false, - name, cma, nid); - if (ret) - pr_warn("%s: reservation failed: err %d, node %d", __func__, - ret, nid); - } + cma = &dma_contiguous_numa_area[nid]; + snprintf(name, sizeof(name), "numa%d", nid); + ret = cma_declare_contiguous_nid(0, size, 0, 0, 0, false, name, cma, nid); + if (ret) + pr_warn("%s: reservation failed: err %d, node %d", __func__, + ret, nid); } } #else @@ -437,16 +426,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) #ifdef CONFIG_DMA_NUMA_CMA if (nid != NUMA_NO_NODE && !(gfp & (GFP_DMA | GFP_DMA32))) { - struct cma *cma = dma_contiguous_pernuma_area[nid]; + struct cma *cma = dma_contiguous_numa_area[nid]; struct page *page; - - if (cma) { - page = cma_alloc_aligned(cma, size, gfp); - if (page) - return page; - } - - cma = dma_contiguous_numa_area[nid]; if (cma) { page = cma_alloc_aligned(cma, size, gfp); if (page) @@ -484,9 +465,6 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size) * otherwise, page is from either per-numa cma or default cma */ #ifdef CONFIG_DMA_NUMA_CMA - if (cma_release(dma_contiguous_pernuma_area[page_to_nid(page)], - page, count)) - return; if (cma_release(dma_contiguous_numa_area[page_to_nid(page)], page, count)) return; From 7318301e060e3f1207a34f2fca6cd86770dd9160 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 2 Jun 2026 20:17:58 -0700 Subject: [PATCH 9/9] dma: map_benchmark: turn dma_sg_map_param buf into a flexible array The buf pointer was kmalloc_array()'d immediately after the parent struct allocation, with the count (granule, validated to 1..1024 by the ioctl) trivially available beforehand. Move buf to the struct tail as a flexible array member and fold the two allocations into a single kzalloc_flex(), dropping the kfree(params->buf) in both the prepare error path and unprepare. Add __counted_by for extra runtime analysis. Assisted-by: Claude:Opus-4.7 Signed-off-by: Rosen Penev Reviewed-by: Qinxin Xia Signed-off-by: Marek Szyprowski Link: https://lore.kernel.org/r/20260603031758.290538-1-rosenp@gmail.com --- kernel/dma/map_benchmark.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index 29eeb5fdf199..fdc070f419f6 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -121,35 +121,35 @@ static struct map_benchmark_ops dma_single_map_benchmark_ops = { struct dma_sg_map_param { struct sg_table sgt; struct device *dev; - void **buf; u32 npages; u32 dma_dir; + void *buf[] __counted_by(npages); }; static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map) { + struct dma_sg_map_param *params; struct scatterlist *sg; + u32 npages; int i; - struct dma_sg_map_param *params = kzalloc(sizeof(*params), GFP_KERNEL); - - if (!params) - return NULL; /* * Set the number of scatterlist entries based on the granule. * In SG mode, 'granule' represents the number of scatterlist entries. * Each scatterlist entry corresponds to a single page. */ - params->npages = map->bparam.granule; + npages = map->bparam.granule; + + params = kzalloc_flex(*params, buf, npages); + if (!params) + return NULL; + + params->npages = npages; params->dma_dir = map->bparam.dma_dir; params->dev = map->dev; - params->buf = kmalloc_array(params->npages, sizeof(*params->buf), - GFP_KERNEL); - if (!params->buf) - goto out; if (sg_alloc_table(¶ms->sgt, params->npages, GFP_KERNEL)) - goto free_buf; + goto free_params; for_each_sgtable_sg(¶ms->sgt, sg, i) { params->buf[i] = (void *)__get_free_page(GFP_KERNEL); @@ -166,9 +166,7 @@ static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map) free_page((unsigned long)params->buf[i]); sg_free_table(¶ms->sgt); -free_buf: - kfree(params->buf); -out: +free_params: kfree(params); return NULL; } @@ -183,7 +181,6 @@ static void dma_sg_map_benchmark_unprepare(void *mparam) sg_free_table(¶ms->sgt); - kfree(params->buf); kfree(params); }