github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/sparse-vmemmap: move vmemmap optimization helpers to a public header

Browse Source 
The vmemmap optimization helpers currently live in mm/sparse.h, which is
an internal MM header.  That works for MM code, but prevents powerpc from
using the same interfaces without including a private header.

Move the declarations and inline helpers to vmemmap-optimization.h.  This
is a preparatory change for powerpc, which has its own vmemmap
optimization implementation and needs to use the common vmemmap
optimization interfaces from architecture code.

Link: https://lore.kernel.org/20260930140627.57431-9-songmuchun@bytedance.com
Signed-off-by: Muchun Song <muchun.song@linux.dev>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Qi Zheng <qi.zheng@linux.dev>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Cc: Oscar Salvador <osalvador@suse.de>
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Christophe Leroy <chleroy@kernel.org>
Cc: Ritesh Harjani (IBM) <ritesh.list@gmail.com>
Cc: Shrikanth Hegde <sshegde@linux.ibm.com>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Lance Yang <lance.yang@linux.dev>
This commit is contained in:
Muchun Song 2026-09-30 22:06:23 +08:00 committed by Andrew Morton
parent ba9164f880
commit d3a699634c
8 changed files with 115 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
MAINTAINERS
View File

@ -12098,6 +12098,7 @@ F: Documentation/mm/hugetlbfs_reserv.rst
F: Documentation/mm/vmemmap_dedup.rst
F: fs/hugetlbfs/
F: include/linux/hugetlb.h
F: include/linux/vmemmap-optimization.h
F: include/trace/events/hugetlbfs.h
F: mm/hugetlb.c
F: mm/hugetlb_cgroup.c

1
arch/loongarch/include/asm/pgtable.h
View File

@ -72,6 +72,7 @@
#include <linux/mm_types.h>
#include <linux/mmzone.h>
#include <linux/vmemmap-optimization.h>
#include <asm/fixmap.h>
#include <asm/sparsemem.h>

1
arch/riscv/mm/init.c
View File

@ -22,6 +22,7 @@
#include <linux/hugetlb.h>
#include <linux/kfence.h>
#include <linux/execmem.h>
#include <linux/vmemmap-optimization.h>
#include <asm/alternative.h>
#include <asm/fixmap.h>

17
include/linux/mmzone.h
View File

@ -96,23 +96,6 @@
#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER)
/*
* HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to
* be naturally aligned with regard to the folio size.
*
* HVO which is only active if the size of struct page is a power of 2.
*/
#define MAX_FOLIO_VMEMMAP_ALIGN \
(IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \
is_power_of_2(sizeof(struct page)) ? \
MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
/* The number of retained vmemmap pages with HVO enabled. */
#define VMEMMAP_OPTIMIZATION_PAGES 1
#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \
(VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page))
#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1)
enum migratetype {
MIGRATE_UNMOVABLE,
MIGRATE_MOVABLE,

109
include/linux/vmemmap-optimization.h Normal file
View File

@ -0,0 +1,109 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* vmemmap-optimization.h
*
* Generic vmemmap optimization declarations.
*
* Author: Muchun Song <songmuchun@bytedance.com>
*/
#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H
#define _LINUX_VMEMMAP_OPTIMIZATION_H
#include <linux/align.h>
#include <linux/log2.h>
#include <linux/mmdebug.h>
#include <linux/mmzone.h>
/*
* HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to
* be naturally aligned with regard to the folio size.
*
* HVO which is only active if the size of struct page is a power of 2.
*/
#define MAX_FOLIO_VMEMMAP_ALIGN \
(IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \
is_power_of_2(sizeof(struct page)) ? \
MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0)
/* The number of retained vmemmap pages with HVO enabled. */
#define VMEMMAP_OPTIMIZATION_PAGES 1
#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \
(VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page))
#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1)
#ifdef CONFIG_VMEMMAP_OPTIMIZATION
static inline unsigned int section_compound_order(const struct mem_section *section)
{
return section->compound_page_order;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
VM_WARN_ON(section_compound_order(section) && order &&
section_compound_order(section) != order);
section->compound_page_order = order;
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
unsigned long section_nr = pfn_to_section_nr(pfn);
if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
return;
for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
section_set_compound_order(__nr_to_section(section_nr + i), order);
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return section_compound_order(__pfn_to_section(pfn));
}
struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone);
#else
static inline unsigned int section_compound_order(const struct mem_section *section)
{
return 0;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return 0;
}
#endif /* CONFIG_VMEMMAP_OPTIMIZATION */
static inline bool vmemmap_optimizable_pfn(unsigned long pfn)
{
const unsigned int order = pfn_to_section_compound_order(pfn);
const unsigned long nr_pages = 1UL << order;
if (!is_power_of_2(sizeof(struct page)))
return false;
return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES;
}
static inline bool vmemmap_optimizable_order(unsigned int order)
{
if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION))
return false;
if (!is_power_of_2(sizeof(struct page)))
return false;
return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER;
}
#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */

2
mm/hugetlb.c
View File

@ -38,6 +38,7 @@
#include <linux/mm_inline.h>
#include <linux/padata.h>
#include <linux/pgalloc.h>
#include <linux/vmemmap-optimization.h>
#include <asm/page.h>
#include <asm/tlb.h>
@ -52,7 +53,6 @@
#include "hugetlb_cma.h"
#include "hugetlb_internal.h"
#include "mm_init.h"
#include "sparse.h"
#include <linux/page-isolation.h>
#define HUGE_BOOTMEM_ZONES_VALID BIT(0)

2
mm/hugetlb_vmemmap.c
View File

@ -15,10 +15,10 @@
#include <linux/mmdebug.h>
#include <linux/pagewalk.h>
#include <linux/pgalloc.h>
#include <linux/vmemmap-optimization.h>
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
#include "sparse.h"
/**
* struct vmemmap_remap_walk - walk vmemmap page table

78
mm/sparse.h
View File

@ -9,80 +9,7 @@
#define __MM_SPARSE_H
#include <linux/mmzone.h>
#ifdef CONFIG_VMEMMAP_OPTIMIZATION
static inline unsigned int section_compound_order(const struct mem_section *section)
{
return section->compound_page_order;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
VM_WARN_ON(section_compound_order(section) && order &&
section_compound_order(section) != order);
section->compound_page_order = order;
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
unsigned long section_nr = pfn_to_section_nr(pfn);
if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))
return;
for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++)
section_set_compound_order(__nr_to_section(section_nr + i), order);
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return section_compound_order(__pfn_to_section(pfn));
}
#else
static inline unsigned int section_compound_order(const struct mem_section *section)
{
return 0;
}
static inline void section_set_compound_order(struct mem_section *section,
unsigned int order)
{
}
static inline void section_set_compound_order_range(unsigned long pfn,
unsigned long nr_pages, unsigned int order)
{
}
static inline unsigned int pfn_to_section_compound_order(unsigned long pfn)
{
return 0;
}
#endif
static inline bool vmemmap_optimizable_pfn(unsigned long pfn)
{
const unsigned int order = pfn_to_section_compound_order(pfn);
const unsigned long nr_pages = 1UL << order;
if (!is_power_of_2(sizeof(struct page)))
return false;
return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES;
}
static inline bool vmemmap_optimizable_order(unsigned int order)
{
if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION))
return false;
if (!is_power_of_2(sizeof(struct page)))
return false;
return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER;
}
#include <linux/vmemmap-optimization.h>
/*
* mm/sparse.c
@ -142,9 +69,6 @@ static inline void sparse_sections_init(void) {}
* mm/sparse-vmemmap.c
*/
#ifdef CONFIG_SPARSEMEM_VMEMMAP
#ifdef CONFIG_VMEMMAP_OPTIMIZATION
struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone);
#endif
void sparse_init_subsection_map(void);
int section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
struct vmem_altmap *altmap, struct dev_pagemap *pgmap);