github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 04:34:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm: preserve RWP marker across PTE rewrites

Browse Source 
The uffd PTE bit must survive any kernel path that rewrites a PTE on a
VM_UFFD_RWP VMA, otherwise the marker that carries PAGE_NONE semantics is
silently dropped and the next access leaks past RWP tracking.  Wire the
preservation through every path that rewrites a VM_UFFD_RWP PTE.

Swap and device-exclusive: do_swap_page(), restore_exclusive_pte(), and
unuse_pte() (swapoff()) re-apply PAGE_NONE when the swap PTE carries the
uffd bit and the VMA has VM_UFFD_RWP.

Migration: remove_migration_pte() and remove_migration_pmd() do the same
after the migration entry is replaced with a real PTE/PMD.

Fork: __copy_present_ptes(), copy_present_page(), copy_nonpresent_pte(),
copy_huge_pmd(), copy_huge_non_present_pmd(), and
copy_hugetlb_page_range() keep the uffd bit on the child when the
destination VMA has VM_UFFD_RWP, matching the existing VM_UFFD_WP
handling.  Add VM_UFFD_RWP to VM_COPY_ON_FORK so the flag itself
propagates.

mprotect(): change_pte_range() and change_huge_pmd() restore PAGE_NONE
after pte_modify()/pmd_modify() have recomputed the base protection from a
(possibly user-changed) vm_page_prot.  pte_modify() preserves _PAGE_UFFD,
so the bit stays; we just have to force PAGE_NONE back on top.

Link: https://lore.kernel.org/20260708111417.173443-8-kirill@shutemov.name
Signed-off-by: Kiryl Shutsemau <kas@kernel.org>
Assisted-by: Claude:claude-opus-4-6
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: David Hildenbrand <david@kernel.org>
Cc: James Houghton <jthoughton@google.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Liam Howlett <liam@infradead.org>
Cc: Lorenzo Stoakes <ljs@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sean Christopherson <seanjc@google.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Kiryl Shutsemau (Meta) 2026-07-08 12:14:08 +01:00 committed by Andrew Morton
parent 9cf3c554ac
commit 763f43865f
9 changed files with 199 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/linux/mm.h
View File

@ -684,7 +684,8 @@ enum {
* only and thus cannot be reconstructed on page
* fault.
*/
#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)
#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \
VM_MAYBE_GUARD)
/*
* mapping from the currently active vm_flags protection bits (the

50
mm/huge_memory.c
View File

@ -1856,7 +1856,7 @@ static void copy_huge_non_present_pmd(
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
mm_inc_nr_ptes(dst_mm);
pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
if (!userfaultfd_wp(dst_vma))
if (!userfaultfd_protected(dst_vma))
pmd = pmd_swp_clear_uffd(pmd);
set_pmd_at(dst_mm, addr, dst_pmd, pmd);
}
@ -1951,9 +1951,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
out_zero_page:
mm_inc_nr_ptes(dst_mm);
pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
pmdp_set_wrprotect(src_mm, addr, src_pmd);
if (!userfaultfd_wp(dst_vma))
/* See __copy_present_ptes(): restore accessible protection. */
if (!userfaultfd_protected(dst_vma)) {
if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd))
pmd = pmd_modify(pmd, dst_vma->vm_page_prot);
pmd = pmd_clear_uffd(pmd);
}
pmdp_set_wrprotect(src_mm, addr, src_pmd);
pmd = pmd_wrprotect(pmd);
set_pmd:
pmd = pmd_mkold(pmd);
@ -2539,8 +2545,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
}
pmd = move_soft_dirty_pmd(pmd);
if (vma_has_uffd_without_event_remap(vma))
if (vma_has_uffd_without_event_remap(vma)) {
/*
* See __copy_present_ptes(): normalise the RWP marker
* so the destination starts accessible instead of
* taking a numa-hinting fault on first access. Only the
* marker (protnone + uffd) needs it; leave other present
* PMDs in the VMA untouched.
*/
if (pmd_present(pmd) && userfaultfd_rwp(vma) &&
pmd_uffd(pmd))
pmd = pmd_modify(pmd, vma->vm_page_prot);
pmd = clear_uffd_wp_pmd(pmd);
}
set_pmd_at(mm, new_addr, new_pmd, pmd);
if (force_flush)
flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
@ -2680,6 +2697,10 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
*/
entry = pmd_clear_uffd(entry);
/* See change_pte_range(): preserve RWP protection across mprotect() */
if (userfaultfd_rwp(vma) && pmd_uffd(entry))
entry = pmd_modify(entry, PAGE_NONE);
/* See change_pte_range(). */
if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
can_change_pmd_writable(vma, addr, entry))
@ -2847,6 +2868,13 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm
_dst_pmd = move_soft_dirty_pmd(src_pmdval);
_dst_pmd = clear_uffd_wp_pmd(_dst_pmd);
}
/* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */
if (userfaultfd_rwp(dst_vma)) {
_dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE);
_dst_pmd = pmd_mkuffd(_dst_pmd);
}
set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd);
src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd);
@ -3023,6 +3051,11 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
entry = pte_mkspecial(entry);
if (pmd_uffd(old_pmd))
entry = pte_mkuffd(entry);
/* Restore PAGE_NONE so an RWP marker keeps trapping */
if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd))
entry = pte_modify(entry, PAGE_NONE);
VM_BUG_ON(!pte_none(ptep_get(pte)));
set_pte_at(mm, addr, pte, entry);
pte++;
@ -3299,6 +3332,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
if (uffd_wp)
entry = pte_mkuffd(entry);
/* Restore PAGE_NONE so an RWP marker keeps trapping */
if (userfaultfd_rwp(vma) && uffd_wp)
entry = pte_modify(entry, PAGE_NONE);
for (i = 0; i < HPAGE_PMD_NR; i++)
VM_WARN_ON(!pte_none(ptep_get(pte + i)));
@ -4989,6 +5026,11 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
pmde = pmd_mkwrite(pmde, vma);
if (pmd_swp_uffd(*pvmw->pmd))
pmde = pmd_mkuffd(pmde);
/* See do_swap_page(): restore PAGE_NONE for RWP */
if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma))
pmde = pmd_modify(pmde, PAGE_NONE);
if (!softleaf_is_migration_young(entry))
pmde = pmd_mkold(pmde);
/* NOTE: this may contain setting soft-dirty on some archs */

51
mm/hugetlb.c
View File

@ -4871,8 +4871,16 @@ hugetlb_install_folio(struct vm_area_struct *vma, pte_t *ptep, unsigned long add
__folio_mark_uptodate(new_folio);
hugetlb_add_new_anon_rmap(new_folio, vma, addr);
if (userfaultfd_wp(vma) && huge_pte_uffd(old))
if (userfaultfd_protected(vma) && huge_pte_uffd(old)) {
newpte = huge_pte_mkuffd(newpte);
/* Restore PAGE_NONE so the RWP marker keeps trapping. */
if (userfaultfd_rwp(vma)) {
unsigned int shift = huge_page_shift(hstate_vma(vma));
newpte = huge_pte_modify(newpte, PAGE_NONE);
newpte = arch_make_huge_pte(newpte, shift, vma->vm_flags);
}
}
set_huge_pte_at(vma->vm_mm, addr, ptep, newpte, sz);
hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
folio_set_hugetlb_migratable(new_folio);
@ -4963,11 +4971,11 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
softleaf = make_readable_migration_entry(
swp_offset(softleaf));
entry = swp_entry_to_pte(softleaf);
if (userfaultfd_wp(src_vma) && uffd)
if (userfaultfd_protected(src_vma) && uffd)
entry = pte_swp_mkuffd(entry);
set_huge_pte_at(src, addr, src_pte, entry, sz);
}
if (!userfaultfd_wp(dst_vma))
if (!userfaultfd_protected(dst_vma))
entry = pte_swp_clear_uffd(entry);
set_huge_pte_at(dst, addr, dst_pte, entry, sz);
} else if (unlikely(pte_is_marker(entry))) {
@ -5032,6 +5040,16 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
goto next;
}
/* See __copy_present_ptes(): restore accessible protection. */
if (!userfaultfd_protected(dst_vma)) {
if (userfaultfd_rwp(src_vma) && huge_pte_uffd(entry)) {
entry = huge_pte_modify(entry, dst_vma->vm_page_prot);
entry = arch_make_huge_pte(entry, huge_page_shift(h),
dst_vma->vm_flags);
}
entry = huge_pte_clear_uffd(entry);
}
if (cow) {
/*
* No need to notify as we are downgrading page
@ -5044,9 +5062,6 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
entry = huge_pte_wrprotect(entry);
}
if (!userfaultfd_wp(dst_vma))
entry = huge_pte_clear_uffd(entry);
set_huge_pte_at(dst, addr, dst_pte, entry, sz);
hugetlb_count_add(npages, dst);
}
@ -5092,10 +5107,23 @@ static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
huge_pte_clear(mm, new_addr, dst_pte, sz);
} else {
if (need_clear_uffd_wp) {
if (pte_present(pte))
if (pte_present(pte)) {
/*
* See __copy_present_ptes(): normalise the RWP
* marker so the destination starts accessible
* instead of taking a numa-hinting fault on
* first access. Only the marker (protnone + uffd)
* needs it; leave other present PTEs untouched.
*/
if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
pte = huge_pte_modify(pte, vma->vm_page_prot);
pte = arch_make_huge_pte(pte, huge_page_shift(h),
vma->vm_flags);
}
pte = huge_pte_clear_uffd(pte);
else
} else {
pte = pte_swp_clear_uffd(pte);
}
}
set_huge_pte_at(mm, new_addr, dst_pte, pte, sz);
}
@ -6547,6 +6575,13 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
pte = huge_pte_mkuffd(pte);
else if (uffd_wp_resolve || uffd_rwp_resolve)
pte = huge_pte_clear_uffd(pte);
/* Preserve RWP protection across mprotect() */
if (userfaultfd_rwp(vma) && huge_pte_uffd(pte)) {
pte = huge_pte_modify(pte, PAGE_NONE);
pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
}
huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
pages++;
tlb_remove_huge_tlb_entry(h, &tlb, ptep, address);

66
mm/memory.c
View File

@ -896,6 +896,10 @@ static void restore_exclusive_pte(struct vm_area_struct *vma,
if (pte_swp_uffd(orig_pte))
pte = pte_mkuffd(pte);
/* See do_swap_page(): restore PAGE_NONE for RWP */
if (pte_swp_uffd(orig_pte) && userfaultfd_rwp(vma))
pte = pte_modify(pte, PAGE_NONE);
if ((vma->vm_flags & VM_WRITE) &&
can_change_pte_writable(vma, address, pte)) {
if (folio_test_dirty(folio))
@ -1034,7 +1038,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
make_pte_marker(marker));
return 0;
}
if (!userfaultfd_wp(dst_vma))
if (!userfaultfd_protected(dst_vma))
pte = pte_swp_clear_uffd(pte);
set_pte_at(dst_mm, addr, dst_pte, pte);
return 0;
@ -1081,9 +1085,13 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma
/* All done, just insert the new page copy in the child */
pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot);
pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
/* Uffd-wp needs to be delivered to dest pte as well */
if (userfaultfd_protected(dst_vma) && pte_uffd(ptep_get(src_pte))) {
/* The uffd bit needs to be delivered to the dest pte as well */
pte = pte_mkuffd(pte);
/* Restore PAGE_NONE so the RWP marker keeps trapping */
if (userfaultfd_rwp(dst_vma))
pte = pte_modify(pte, PAGE_NONE);
}
set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
return 0;
}
@ -1093,9 +1101,31 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
pte_t pte, unsigned long addr, int nr)
{
struct mm_struct *src_mm = src_vma->vm_mm;
bool writable;
/*
* Snapshot writability before the RWP-disarm rewrite below: when the
* child is not RWP-armed, pte_modify(pte, dst_vma->vm_page_prot) can
* silently drop _PAGE_RW from a resolved (no-marker) writable PTE,
* so a later pte_write(pte) check would skip the COW wrprotect and
* leave the parent writable over a folio shared with the child.
*/
writable = pte_write(pte);
/*
* Child is not RWP-armed: restore accessible protection so the
* inherited PAGE_NONE does not cost a fault on first read. Gate on
* pte_uffd(pte) so unrelated PAGE_NONE markers (e.g. NUMA balancing)
* are not normalised away.
*/
if (!userfaultfd_protected(dst_vma)) {
if (userfaultfd_rwp(src_vma) && pte_uffd(pte))
pte = pte_modify(pte, dst_vma->vm_page_prot);
pte = pte_clear_uffd(pte);
}
/* If it's a COW mapping, write protect it both processes. */
if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
if (is_cow_mapping(src_vma->vm_flags) && writable) {
wrprotect_ptes(src_mm, addr, src_pte, nr);
pte = pte_wrprotect(pte);
}
@ -1105,9 +1135,6 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
pte = pte_mkclean(pte);
pte = pte_mkold(pte);
if (!userfaultfd_wp(dst_vma))
pte = pte_clear_uffd(pte);
set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
}
@ -4747,6 +4774,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
struct swap_info_struct *si = NULL;
rmap_t rmap_flags = RMAP_NONE;
bool exclusive = false;
bool rwp_restore = false;
softleaf_t entry;
pte_t pte;
vm_fault_t ret = 0;
@ -5026,12 +5054,27 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
if (pte_swp_uffd(vmf->orig_pte))
pte = pte_mkuffd(pte);
/*
* A page reclaimed while RWP-protected carries the uffd bit on
* its swap entry. Re-apply PAGE_NONE on swap-in so the first access
* still traps as an RWP fault. pte_modify() preserves _PAGE_UFFD.
*/
if (pte_swp_uffd(vmf->orig_pte) && userfaultfd_rwp(vma)) {
pte = pte_modify(pte, PAGE_NONE);
rwp_restore = true;
}
/*
* Similar logic as in do_wp_page(); however, optimize for pages that
* are certainly exclusive.
*
* Skip the write upgrade for an RWP-restored pte: it must stay
* PROT_NONE so the access retries through the RWP fault path
* (do_uffd_rwp()) rather than being made writable here.
*/
if (exclusive) {
if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
if (!rwp_restore &&
(vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
!pte_needs_soft_dirty_wp(vma, pte)) {
pte = pte_mkwrite(pte, vma);
if (vmf->flags & FAULT_FLAG_WRITE)
@ -5092,7 +5135,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
folio_put(swapcache);
}
if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte)) {
/*
* For an RWP-restored pte, leave it PROT_NONE and let the write
* retry through the RWP fault path; do not COW it here, which would
* drop the marker for a non-exclusive page.
*/
if ((vmf->flags & FAULT_FLAG_WRITE) && !pte_write(pte) && !rwp_restore) {
ret |= do_wp_page(vmf);
if (ret & VM_FAULT_ERROR)
ret &= VM_FAULT_ERROR;

8
mm/migrate.c
View File

@ -330,6 +330,10 @@ static bool try_to_map_unused_to_zeropage(struct page_vma_mapped_walk *pvmw,
if (pte_swp_uffd(old_pte))
newpte = pte_mkuffd(newpte);
/* See remove_migration_pte(): restore PAGE_NONE for RWP */
if (pte_swp_uffd(old_pte) && userfaultfd_rwp(pvmw->vma))
newpte = pte_modify(newpte, PAGE_NONE);
set_pte_at(pvmw->vma->vm_mm, pvmw->address, pvmw->pte, newpte);
dec_mm_counter(pvmw->vma->vm_mm, mm_counter(folio));
@ -399,6 +403,10 @@ static bool remove_migration_pte(struct folio *folio,
else if (pte_swp_uffd(old_pte))
pte = pte_mkuffd(pte);
/* See do_swap_page(): restore PAGE_NONE for RWP */
if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
pte = pte_modify(pte, PAGE_NONE);
if (folio_test_anon(folio) && !softleaf_is_migration_read(entry))
rmap_flags |= RMAP_EXCLUSIVE;

10
mm/mprotect.c
View File

@ -296,6 +296,16 @@ static __always_inline void change_present_ptes(struct mmu_gather *tlb,
else if (uffd_prot_resolve)
ptent = pte_clear_uffd(ptent);
/*
* The uffd bit on a VM_UFFD_RWP VMA carries PROT_NONE
* semantics. If mprotect() or NUMA hinting changed the
* base protection, restore PAGE_NONE so the PTE still
* traps on any access. pte_modify() preserves
* _PAGE_UFFD.
*/
if (userfaultfd_rwp(vma) && pte_uffd(ptent))
ptent = pte_modify(ptent, PAGE_NONE);
/*
* In some writable, shared mappings, we might want
* to catch actual write access -- see

13
mm/mremap.c
View File

@ -296,10 +296,19 @@ static int move_ptes(struct pagetable_move_control *pmc,
pte_clear(mm, new_addr, new_ptep);
else {
if (need_clear_uffd_wp) {
if (pte_present(pte))
if (pte_present(pte)) {
/*
* See __copy_present_ptes(): normalise
* RWP PTEs so the destination starts
* accessible instead of taking a
* numa-hinting fault on first access.
*/
if (userfaultfd_rwp(vma) && pte_uffd(pte))
pte = pte_modify(pte, vma->vm_page_prot);
pte = pte_clear_uffd(pte);
else
} else {
pte = pte_swp_clear_uffd(pte);
}
}
set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
}

5
mm/swapfile.c
View File

@ -2498,6 +2498,11 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
new_pte = pte_mksoft_dirty(new_pte);
if (pte_swp_uffd(old_pte))
new_pte = pte_mkuffd(new_pte);
/* See do_swap_page(): restore PAGE_NONE for RWP */
if (pte_swp_uffd(old_pte) && userfaultfd_rwp(vma))
new_pte = pte_modify(new_pte, PAGE_NONE);
setpte:
set_pte_at(vma->vm_mm, addr, pte, new_pte);
folio_put_swap(swapcache, folio_file_page(swapcache, swp_offset(entry)));

17
mm/userfaultfd.c
View File

@ -1292,6 +1292,13 @@ static long move_present_ptes(struct mm_struct *mm,
if (pte_dirty(orig_src_pte))
orig_dst_pte = pte_mkdirty(orig_dst_pte);
orig_dst_pte = pte_mkwrite(orig_dst_pte, dst_vma);
/* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */
if (userfaultfd_rwp(dst_vma)) {
orig_dst_pte = pte_modify(orig_dst_pte, PAGE_NONE);
orig_dst_pte = pte_mkuffd(orig_dst_pte);
}
set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);
src_addr += PAGE_SIZE;
@ -1373,6 +1380,9 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma,
orig_src_pte = ptep_get_and_clear(mm, src_addr, src_pte);
if (pgtable_supports_soft_dirty())
orig_src_pte = pte_swp_mksoft_dirty(orig_src_pte);
/* Re-arm RWP on the moved swap entry if dst_vma is RWP-registered. */
if (userfaultfd_rwp(dst_vma))
orig_src_pte = pte_swp_mkuffd(orig_src_pte);
set_pte_at(mm, dst_addr, dst_pte, orig_src_pte);
double_pt_unlock(dst_ptl, src_ptl);
@ -1399,6 +1409,13 @@ static int move_zeropage_pte(struct mm_struct *mm,
zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr),
dst_vma->vm_page_prot));
/* Re-arm RWP on the moved PTE if dst_vma is RWP-registered. */
if (userfaultfd_rwp(dst_vma)) {
zero_pte = pte_modify(zero_pte, PAGE_NONE);
zero_pte = pte_mkuffd(zero_pte);
}
ptep_clear_flush(src_vma, src_addr, src_pte);
set_pte_at(mm, dst_addr, dst_pte, zero_pte);
double_pt_unlock(dst_ptl, src_ptl);