github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-26 08:02:27 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

iommu/pages: Move from struct page to struct ioptdesc and folio

Browse Source 
This brings the iommu page table allocator into the modern world of having
its own private page descriptor and not re-using fields from struct page
for its own purpose. It follows the basic pattern of struct ptdesc which
did this transformation for the CPU page table allocator.

Currently iommu-pages is pretty basic so this isn't a huge benefit,
however I see a coming need for features that CPU allocator has, like sub
PAGE_SIZE allocations, and RCU freeing. This provides the base
infrastructure to implement those cleanly.

Remove numa_node_id() calls from the inlines and instead use NUMA_NO_NODE
which will get switched to numa_mem_id(), which seems to be the right ID
to use for memory allocations.

Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Tested-by: Nicolin Chen <nicolinc@nvidia.com>
Tested-by: Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/13-v4-c8663abbb606+3f7-iommu_pages_jgg@nvidia.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
Jason Gunthorpe 2025-04-08 13:54:01 -03:00 committed by Joerg Roedel
parent 27bc9f717f
commit 212fcf36c6
2 changed files with 78 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
drivers/iommu/iommu-pages.c
View File

@ -7,6 +7,21 @@
#include <linux/gfp.h>
#include <linux/mm.h>
#define IOPTDESC_MATCH(pg_elm, elm) \
static_assert(offsetof(struct page, pg_elm) == \
offsetof(struct ioptdesc, elm))
IOPTDESC_MATCH(flags, __page_flags);
IOPTDESC_MATCH(lru, iopt_freelist_elm); /* Ensure bit 0 is clear */
IOPTDESC_MATCH(mapping, __page_mapping);
IOPTDESC_MATCH(private, _private);
IOPTDESC_MATCH(page_type, __page_type);
IOPTDESC_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
IOPTDESC_MATCH(memcg_data, memcg_data);
#endif
#undef IOPTDESC_MATCH
static_assert(sizeof(struct ioptdesc) <= sizeof(struct page));
/**
* iommu_alloc_pages_node - Allocate a zeroed page of a given order from
* specific NUMA node
@ -20,10 +35,17 @@
void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)
{
const unsigned long pgcnt = 1UL << order;
struct page *page;
struct folio *folio;
page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order);
if (unlikely(!page))
/*
* __folio_alloc_node() does not handle NUMA_NO_NODE like
* alloc_pages_node() did.
*/
if (nid == NUMA_NO_NODE)
nid = numa_mem_id();
folio = __folio_alloc_node(gfp | __GFP_ZERO, order, nid);
if (unlikely(!folio))
return NULL;
/*
@ -35,21 +57,21 @@ void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order)
* This is necessary for the proper accounting as IOMMU state can be
* rather large, i.e. multiple gigabytes in size.
*/
mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt);
mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt);
mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt);
lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt);
return page_address(page);
return folio_address(folio);
}
EXPORT_SYMBOL_GPL(iommu_alloc_pages_node);
static void __iommu_free_page(struct page *page)
static void __iommu_free_desc(struct ioptdesc *iopt)
{
unsigned int order = folio_order(page_folio(page));
const unsigned long pgcnt = 1UL << order;
struct folio *folio = ioptdesc_folio(iopt);
const unsigned long pgcnt = 1UL << folio_order(folio);
mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt);
mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt);
put_page(page);
mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt);
lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt);
folio_put(folio);
}
/**
@ -62,7 +84,7 @@ void iommu_free_pages(void *virt)
{
if (!virt)
return;
__iommu_free_page(virt_to_page(virt));
__iommu_free_desc(virt_to_ioptdesc(virt));
}
EXPORT_SYMBOL_GPL(iommu_free_pages);
@ -74,9 +96,9 @@ EXPORT_SYMBOL_GPL(iommu_free_pages);
*/
void iommu_put_pages_list(struct iommu_pages_list *list)
{
struct page *p, *tmp;
struct ioptdesc *iopt, *tmp;
list_for_each_entry_safe(p, tmp, &list->pages, lru)
__iommu_free_page(p);
list_for_each_entry_safe(iopt, tmp, &list->pages, iopt_freelist_elm)
__iommu_free_desc(iopt);
}
EXPORT_SYMBOL_GPL(iommu_put_pages_list);

43
drivers/iommu/iommu-pages.h
View File

@ -9,6 +9,43 @@
#include <linux/iommu.h>
/**
* struct ioptdesc - Memory descriptor for IOMMU page tables
* @iopt_freelist_elm: List element for a struct iommu_pages_list
*
* This struct overlays struct page for now. Do not modify without a good
* understanding of the issues.
*/
struct ioptdesc {
unsigned long __page_flags;
struct list_head iopt_freelist_elm;
unsigned long __page_mapping;
pgoff_t __index;
void *_private;
unsigned int __page_type;
atomic_t __page_refcount;
#ifdef CONFIG_MEMCG
unsigned long memcg_data;
#endif
};
static inline struct ioptdesc *folio_ioptdesc(struct folio *folio)
{
return (struct ioptdesc *)folio;
}
static inline struct folio *ioptdesc_folio(struct ioptdesc *iopt)
{
return (struct folio *)iopt;
}
static inline struct ioptdesc *virt_to_ioptdesc(void *virt)
{
return folio_ioptdesc(virt_to_folio(virt));
}
void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order);
void iommu_free_pages(void *virt);
void iommu_put_pages_list(struct iommu_pages_list *list);
@ -21,7 +58,7 @@ void iommu_put_pages_list(struct iommu_pages_list *list);
static inline void iommu_pages_list_add(struct iommu_pages_list *list,
void *virt)
{
list_add_tail(&virt_to_page(virt)->lru, &list->pages);
list_add_tail(&virt_to_ioptdesc(virt)->iopt_freelist_elm, &list->pages);
}
/**
@ -56,7 +93,7 @@ static inline bool iommu_pages_list_empty(struct iommu_pages_list *list)
*/
static inline void *iommu_alloc_pages(gfp_t gfp, int order)
{
return iommu_alloc_pages_node(numa_node_id(), gfp, order);
return iommu_alloc_pages_node(NUMA_NO_NODE, gfp, order);
}
/**
@ -79,7 +116,7 @@ static inline void *iommu_alloc_page_node(int nid, gfp_t gfp)
*/
static inline void *iommu_alloc_page(gfp_t gfp)
{
return iommu_alloc_pages_node(numa_node_id(), gfp, 0);
return iommu_alloc_pages_node(NUMA_NO_NODE, gfp, 0);
}
#endif /* __IOMMU_PAGES_H */