github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 04:34:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm: add MM_CP_UFFD_RWP change_protection() flag

Browse Source 
Preparatory patch.  Add the change_protection() primitive that userfaultfd
RWP will use.

An RWP-protected PTE is PAGE_NONE with the uffd PTE bit set.  The
PROT_NONE half makes the CPU fault on any access; the uffd bit
distinguishes an RWP fault from a plain mprotect(PROT_NONE) or NUMA
hinting fault.  MM_CP_UFFD_WP and MM_CP_UFFD_RWP share the same PTE bit,
so the two cannot be used together on the same range.

Two new change_protection() flags:

  MM_CP_UFFD_RWP            install PAGE_NONE and set the uffd bit
  MM_CP_UFFD_RWP_RESOLVE    restore vma->vm_page_prot, clear the uffd bit

Both are wired through change_pte_range(), change_huge_pmd(), and
hugetlb_change_protection() so anon, shmem, THP, and hugetlb all share the
same semantics.

Link: https://lore.kernel.org/20260708111417.173443-7-kirill@shutemov.name
Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Assisted-by: Claude:claude-opus-4-6
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Reviewed-by: SeongJae Park <sj@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: James Houghton <jthoughton@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kiryl Shutsemau (Meta) 2026-07-08 12:14:07 +01:00 committed by Andrew Morton
parent 134c1d031a
commit 9cf3c554ac
5 changed files with 81 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/linux/mm.h
View File

@ -3340,6 +3340,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
#define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */
#define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \
MM_CP_UFFD_WP_RESOLVE)
/* Whether this change is for uffd RWP */
#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */
#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */
#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \
MM_CP_UFFD_RWP_RESOLVE)
bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
pte_t pte);

1
include/linux/userfaultfd_k.h
View File

@ -369,7 +369,6 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma,
return false;
}
static inline bool userfaultfd_armed(struct vm_area_struct *vma)
{
return false;

36
mm/huge_memory.c
View File

@ -2553,8 +2553,8 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
}
static void change_non_present_huge_pmd(struct mm_struct *mm,
unsigned long addr, pmd_t *pmd, bool uffd_wp,
bool uffd_wp_resolve)
unsigned long addr, pmd_t *pmd, bool uffd_prot,
bool uffd_prot_resolve)
{
softleaf_t entry = softleaf_from_pmd(*pmd);
pmd_t newpmd;
@ -2583,9 +2583,9 @@ static void change_non_present_huge_pmd(struct mm_struct *mm,
newpmd = *pmd;
}
if (uffd_wp)
if (uffd_prot)
newpmd = pmd_swp_mkuffd(newpmd);
else if (uffd_wp_resolve)
else if (uffd_prot_resolve)
newpmd = pmd_swp_clear_uffd(newpmd);
if (!pmd_same(*pmd, newpmd))
set_pmd_at(mm, addr, pmd, newpmd);
@ -2606,8 +2606,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
spinlock_t *ptl;
pmd_t oldpmd, entry;
bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
bool uffd_prot_resolve = cp_flags &
(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
int ret = 1;
tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
@ -2620,11 +2621,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
return 0;
if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) {
change_non_present_huge_pmd(mm, addr, pmd, uffd_wp,
uffd_wp_resolve);
change_non_present_huge_pmd(mm, addr, pmd, uffd_prot,
uffd_prot_resolve);
goto unlock;
}
/* Already in the desired state */
if (prot_numa && pmd_protnone(*pmd))
goto unlock;
if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd))
goto unlock;
if (prot_numa) {
/*
@ -2635,9 +2642,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
if (is_huge_zero_pmd(*pmd))
goto unlock;
if (pmd_protnone(*pmd))
goto unlock;
if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma,
vma_is_single_threaded_private(vma)))
goto unlock;
@ -2666,9 +2670,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
oldpmd = pmdp_invalidate_ad(vma, addr, pmd);
entry = pmd_modify(oldpmd, newprot);
if (uffd_wp)
if (uffd_prot)
entry = pmd_mkuffd(entry);
else if (uffd_wp_resolve)
else if (uffd_prot_resolve)
/*
* Leave the write bit to be handled by PF interrupt
* handler, then things like COW could be properly
@ -2714,10 +2718,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
return 1;
/*
* Huge entries on userfault-wp only works with anonymous, while we
* don't have anonymous PUDs yet.
* Huge entries on userfault-wp or userfault-rwp only work with
* anonymous, while we don't have anonymous PUDs yet.
*/
if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL))
if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL)))
return 1;
ptl = __pud_trans_huge_lock(pudp, vma);

25
mm/hugetlb.c
View File

@ -6427,6 +6427,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
unsigned long last_addr_mask;
bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
bool uffd_rwp_resolve = cp_flags & MM_CP_UFFD_RWP_RESOLVE;
struct mmu_gather tlb;
/*
@ -6452,6 +6454,11 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
ptep = hugetlb_walk(vma, address, psize);
if (!ptep) {
/*
* uffd_wp installs a pte marker on the unpopulated
* entry; uffd_rwp does not install markers so the
* allocation is unnecessary for it.
*/
if (!uffd_wp) {
address |= last_addr_mask;
continue;
@ -6473,7 +6480,8 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
* shouldn't happen at all. Warn about it if it
* happened due to some reason.
*/
WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
WARN_ON_ONCE(uffd_wp || uffd_wp_resolve ||
uffd_rwp || uffd_rwp_resolve);
pages++;
spin_unlock(ptl);
address |= last_addr_mask;
@ -6507,9 +6515,9 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
pages++;
}
if (uffd_wp)
if (uffd_wp || uffd_rwp)
newpte = pte_swp_mkuffd(newpte);
else if (uffd_wp_resolve)
else if (uffd_wp_resolve || uffd_rwp_resolve)
newpte = pte_swp_clear_uffd(newpte);
if (!pte_same(pte, newpte))
set_huge_pte_at(mm, address, ptep, newpte, psize);
@ -6520,19 +6528,24 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
* pte_marker_uffd_wp()==true implies !poison
* because they're mutual exclusive.
*/
if (pte_is_uffd_wp_marker(pte) && uffd_wp_resolve)
if (pte_is_uffd_wp_marker(pte) &&
(uffd_wp_resolve || uffd_rwp_resolve))
/* Safe to modify directly (non-present->none). */
huge_pte_clear(mm, address, ptep, psize);
} else {
pte_t old_pte;
unsigned int shift = huge_page_shift(hstate_vma(vma));
/* Already protnone with uffd bit set? Nothing to do. */
if (uffd_rwp && pte_protnone(pte) && huge_pte_uffd(pte))
goto next;
old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
pte = huge_pte_modify(old_pte, newprot);
pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
if (uffd_wp)
if (uffd_wp || uffd_rwp)
pte = huge_pte_mkuffd(pte);
else if (uffd_wp_resolve)
else if (uffd_wp_resolve || uffd_rwp_resolve)
pte = huge_pte_clear_uffd(pte);
huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
pages++;

48
mm/mprotect.c
View File

@ -214,8 +214,9 @@ static __always_inline void set_write_prot_commit_flush_ptes(struct vm_area_stru
static long change_softleaf_pte(struct vm_area_struct *vma,
unsigned long addr, pte_t *pte, pte_t oldpte, unsigned long cp_flags)
{
const bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
const bool uffd_prot_resolve = cp_flags &
(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
softleaf_t entry = softleaf_from_pte(oldpte);
pte_t newpte;
@ -256,7 +257,7 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
* to unprotect it, drop it; the next page
* fault will trigger without uffd trapping.
*/
if (uffd_wp_resolve) {
if (uffd_prot_resolve) {
pte_clear(vma->vm_mm, addr, pte);
return 1;
}
@ -265,9 +266,9 @@ static long change_softleaf_pte(struct vm_area_struct *vma,
newpte = oldpte;
}
if (uffd_wp)
if (uffd_prot)
newpte = pte_swp_mkuffd(newpte);
else if (uffd_wp_resolve)
else if (uffd_prot_resolve)
newpte = pte_swp_clear_uffd(newpte);
if (!pte_same(oldpte, newpte)) {
@ -282,16 +283,17 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb,
int nr_ptes, unsigned long end, pgprot_t newprot,
struct folio *folio, struct page *page, unsigned long cp_flags)
{
const bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
const bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
const bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP);
const bool uffd_prot_resolve = cp_flags &
(MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE);
pte_t ptent, oldpte;
oldpte = modify_prot_start_ptes(vma, addr, ptep, nr_ptes);
ptent = pte_modify(oldpte, newprot);
if (uffd_wp)
if (uffd_prot)
ptent = pte_mkuffd(ptent);
else if (uffd_wp_resolve)
else if (uffd_prot_resolve)
ptent = pte_clear_uffd(ptent);
/*
@ -325,6 +327,7 @@ static long change_pte_range(struct mmu_gather *tlb,
long pages = 0;
bool is_private_single_threaded;
bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
bool uffd_rwp = cp_flags & MM_CP_UFFD_RWP;
bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
int nr_ptes;
@ -350,6 +353,14 @@ static long change_pte_range(struct mmu_gather *tlb,
/* Already in the desired state. */
if (prot_numa && pte_protnone(oldpte))
continue;
/*
* RWP-protected PTEs carry _PAGE_UFFD as a marker on
* top of PROT_NONE. Skip only entries already in that
* exact state; plain PROT_NONE from mprotect() still needs
* to be promoted so future faults can be distinguished.
*/
if (uffd_rwp && pte_protnone(oldpte) && pte_uffd(oldpte))
continue;
page = vm_normal_page(vma, addr, oldpte);
if (page)
@ -358,6 +369,8 @@ static long change_pte_range(struct mmu_gather *tlb,
/*
* Avoid trapping faults against the zero or KSM
* pages. See similar comment in change_huge_pmd.
* Skip this filter for uffd RWP which
* must set protnone regardless of NUMA placement.
*/
if (prot_numa &&
!folio_can_map_prot_numa(folio, vma,
@ -428,7 +441,7 @@ pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
* (e.g. 2M shmem) because file thp is handled differently when
* split by erasing the pmd so far.
*/
return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
return (cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP)) && !vma_is_anonymous(vma);
}
/*
@ -667,7 +680,16 @@ long change_protection(struct mmu_gather *tlb,
pgprot_t newprot = vma->vm_page_prot;
long pages;
BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
/*
* MM_CP_UFFD_{WP,RWP} and _RESOLVE are mutually exclusive within one
* change, and WP and RWP cannot mix. Miswired callers get a warn and
* a no-op; userspace cannot reach this state.
*/
if (WARN_ON_ONCE((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL ||
(cp_flags & MM_CP_UFFD_RWP_ALL) == MM_CP_UFFD_RWP_ALL ||
((cp_flags & MM_CP_UFFD_WP_ALL) &&
(cp_flags & MM_CP_UFFD_RWP_ALL))))
return 0;
#ifdef CONFIG_NUMA_BALANCING
/*
@ -681,6 +703,10 @@ long change_protection(struct mmu_gather *tlb,
WARN_ON_ONCE(cp_flags & MM_CP_PROT_NUMA);
#endif
if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE) &&
(cp_flags & MM_CP_UFFD_RWP))
newprot = PAGE_NONE;
if (is_vm_hugetlb_page(vma))
pages = hugetlb_change_protection(vma, start, end, newprot,
cp_flags);