github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 12:44:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

dma-direct: swiotlb: handle swiotlb alloc/free outside __dma_direct_alloc_pages

Browse Source 
Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in
dma_direct_alloc() / dma_direct_alloc_pages().

This is needed for follow-up changes that simplify the handling of
memory encryption/decryption based on the DMA attribute flags.

swiotlb backing pages are already mapped decrypted by
swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so
dma-direct should not call dma_set_decrypted() on allocation nor
dma_set_encrypted() on free for swiotlb-backed memory.

Update alloc/free paths to detect swiotlb-backed pages and skip
encrypt/decrypt transitions for those paths. Keep the existing highmem
rejection in dma_direct_alloc_pages() for swiotlb allocations.

Only for "restricted-dma-pool", we currently set `for_alloc = true`, while
rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is
typically used together with "shared-dma-pool", where the shared region is
accessed after remap/ioremap and the returned address is suitable for
decrypted memory access. So existing code paths remain valid.

Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Link: https://lore.kernel.org/r/20260717180442.110954-8-aneesh.kumar@kernel.org
Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
This commit is contained in:
Aneesh Kumar K.V (Arm) 2026-07-17 23:34:25 +05:30 committed by Marek Szyprowski
parent f38ab0d7f5
commit a7139dbbb1
3 changed files with 65 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/swiotlb.h
View File

@ -284,6 +284,8 @@ extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
struct page *swiotlb_alloc(struct device *dev, size_t size);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
void swiotlb_free_from_pool(struct device *dev,
phys_addr_t tlb_addr, struct io_tlb_pool *pool);
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
@ -299,6 +301,10 @@ static inline bool swiotlb_free(struct device *dev, struct page *page,
{
return false;
}
static inline void swiotlb_free_from_pool(struct device *dev,
phys_addr_t tlb_addr, struct io_tlb_pool *pool)
{
}
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
return false;

71
kernel/dma/direct.c
View File

@ -96,14 +96,6 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
static void __dma_direct_free_pages(struct device *dev, struct page *page,
size_t size)
{
if (swiotlb_free(dev, page, size))
return;
dma_free_contiguous(dev, page, size);
}
static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
{
struct page *page = swiotlb_alloc(dev, size);
@ -125,9 +117,6 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
WARN_ON_ONCE(!PAGE_ALIGNED(size));
if (is_swiotlb_for_alloc(dev))
return dma_direct_alloc_swiotlb(dev, size);
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
page = dma_alloc_contiguous(dev, size, gfp);
if (page) {
@ -203,6 +192,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
bool mark_mem_decrypt = true;
struct page *page;
void *ret;
@ -252,11 +242,21 @@ void *dma_direct_alloc(struct device *dev, size_t size,
return page ? ret : NULL;
}
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
if (page) {
mark_mem_decrypt = false;
goto setup_page;
}
return NULL;
}
/* we always manually zero the memory once we are done */
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
if (!page)
return NULL;
setup_page:
/*
* dma_alloc_contiguous can return highmem pages depending on a
* combination the cma= arguments and per-arch setup. These need to be
@ -283,7 +283,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
goto out_free_pages;
} else {
ret = page_address(page);
if (dma_set_decrypted(dev, ret, size))
if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
}
@ -300,10 +300,11 @@ void *dma_direct_alloc(struct device *dev, size_t size,
return ret;
out_encrypt_pages:
if (dma_set_encrypted(dev, page_address(page), size))
if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
return NULL;
out_free_pages:
__dma_direct_free_pages(dev, page, size);
if (!swiotlb_free(dev, page, size))
dma_free_contiguous(dev, page, size);
return NULL;
out_leak_pages:
return NULL;
@ -312,6 +313,9 @@ void *dma_direct_alloc(struct device *dev, size_t size,
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
phys_addr_t phys;
bool mark_mem_encrypted = true;
struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
@ -340,16 +344,25 @@ void dma_direct_free(struct device *dev, size_t size,
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
return;
phys = dma_to_phys(dev, dma_addr);
swiotlb_pool = swiotlb_find_pool(dev, phys);
if (swiotlb_pool)
/* Swiotlb doesn't need a page attribute update on free */
mark_mem_encrypted = false;
if (is_vmalloc_addr(cpu_addr)) {
vunmap(cpu_addr);
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
if (dma_set_encrypted(dev, cpu_addr, size))
if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
return;
}
__dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);
if (swiotlb_pool)
swiotlb_free_from_pool(dev, phys, swiotlb_pool);
else
dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
}
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
@ -361,6 +374,15 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
if (!page)
return NULL;
ret = page_address(page);
goto setup_page;
}
page = __dma_direct_alloc_pages(dev, size, gfp, false);
if (!page)
return NULL;
@ -368,6 +390,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
ret = page_address(page);
if (dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
setup_page:
memset(ret, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return page;
@ -379,16 +402,28 @@ void dma_direct_free_pages(struct device *dev, size_t size,
struct page *page, dma_addr_t dma_addr,
enum dma_data_direction dir)
{
phys_addr_t phys;
void *vaddr = page_address(page);
struct io_tlb_pool *swiotlb_pool;
bool mark_mem_encrypted = true;
/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool_page(dev, page, size))
return;
if (dma_set_encrypted(dev, vaddr, size))
phys = page_to_phys(page);
swiotlb_pool = swiotlb_find_pool(dev, phys);
if (swiotlb_pool)
mark_mem_encrypted = false;
if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size))
return;
__dma_direct_free_pages(dev, page, size);
if (swiotlb_pool)
swiotlb_free_from_pool(dev, phys, swiotlb_pool);
else
dma_free_contiguous(dev, page, size);
}
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \

6
kernel/dma/swiotlb.c
View File

@ -1815,6 +1815,12 @@ bool swiotlb_free(struct device *dev, struct page *page, size_t size)
return true;
}
void swiotlb_free_from_pool(struct device *dev,
phys_addr_t tlb_addr, struct io_tlb_pool *pool)
{
swiotlb_release_slots(dev, tlb_addr, pool);
}
static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
struct device *dev)
{